diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index 82f5b22a8aad9..72da09398ebaf 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -10,8 +10,6 @@ docs: - '**/*.md' tests: - added|modified: '**/src/test/java/**/*.java' -cpp: - - 'pulsar-client-cpp/**' need_owasp: - 'pom.xml' - '**/pom.xml' diff --git a/.github/workflows/ci-cpp-build.yaml b/.github/workflows/ci-cpp-build.yaml deleted file mode 100644 index 7a450efe9725d..0000000000000 --- a/.github/workflows/ci-cpp-build.yaml +++ /dev/null @@ -1,259 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - CPP build -on: - pull_request: - branches: - - master - paths: - - '.github/workflows/**' - - 'pulsar-client-cpp/**' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - changed_files_job: - name: Preconditions - runs-on: ubuntu-20.04 - outputs: - docs_only: ${{ needs.changed_files_job.outputs.docs_only }} - cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }} - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: | - echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Check if the PR has been approved for testing - if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} - env: - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - run: | - build/pulsar_ci_tool.sh check_ready_to_test - - cpp-build-centos7: - needs: changed_files_job - name: Build CPP Client on CentOS7 - runs-on: ubuntu-20.04 - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: build cpp client on centos 7 - run: | - echo "Build C++ client library on CentOS 7" - pulsar-client-cpp/docker-build-centos7.sh - - cpp-build-windows: - needs: changed_files_job - timeout-minutes: 120 - name: Build CPP Client on ${{ matrix.name }} - runs-on: ${{ matrix.os }} - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - env: - VCPKG_ROOT: '${{ github.workspace }}/vcpkg' - strategy: - fail-fast: false - matrix: - include: - - name: 'Windows x64' - os: windows-2022 - triplet: x64-windows - vcpkg_dir: 'C:\vcpkg' - suffix: 'windows-win64' - generator: 'Visual Studio 17 2022' - arch: '-A x64' - - name: 'Windows x86' - os: windows-2022 - triplet: x86-windows - vcpkg_dir: 'C:\vcpkg' - suffix: 'windows-win32' - generator: 'Visual Studio 17 2022' - arch: '-A Win32' - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Restore vcpkg and its artifacts. - uses: actions/cache@v2 - id: vcpkg-cache - with: - path: | - ${{ env.VCPKG_ROOT }} - pulsar-client-cpp/vcpkg_installed - !${{ env.VCPKG_ROOT }}/.git - !${{ env.VCPKG_ROOT }}/buildtrees - !${{ env.VCPKG_ROOT }}/packages - !${{ env.VCPKG_ROOT }}/downloads - key: | - ${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'pulsar-client-cpp/vcpkg.json' ) }} - - - name: Get vcpkg(windows) - if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} - run: | - cd ${{ github.workspace }} - mkdir build -force - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - - - name: remove system vcpkg(windows) - if: runner.os == 'Windows' - run: rm -rf "$VCPKG_INSTALLATION_ROOT" - shell: bash - - - name: Install vcpkg packages - run: | - cd pulsar-client-cpp && ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }} - - - name: Configure (default) - - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd pulsar-client-cpp && \ - cmake \ - -B ./build-0 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET=${{ matrix.triplet }} \ - -DCMAKE_BUILD_TYPE=Release \ - -S . - fi - - - name: Compile - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd pulsar-client-cpp && \ - cmake --build ./build-0 --parallel --config Release - fi - - - name: Configure (dynamic library only) - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd pulsar-client-cpp && \ - cmake \ - -B ./build-1 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET=${{ matrix.triplet }} \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_STATIC_LIB=OFF \ - -S . - fi - - - name: Compile - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd pulsar-client-cpp && \ - cmake --build ./build-1 --parallel --config Release - fi - cpp-deb-rpm-packaging: - needs: changed_files_job - name: Build CPP Client on RPM - runs-on: ubuntu-20.04 - timeout-minutes: 120 - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - run: | - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: Package Pulsar source - run: mvn -B -ntp -q clean package -pl pulsar-client-api -am -DskipTests - - - name: Build Debian packages - run: | - echo "Build Debian packages" - BUILD_IMAGE=1 pulsar-client-cpp/pkg/deb/docker-build-deb.sh - - - name: Build RPM packages - run: | - echo "Build RPM packages" - BUILD_IMAGE=1 pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh - - build-python-wheel: - needs: changed_files_job - name: Build Python Client - runs-on: ubuntu-20.04 - timeout-minutes: 120 - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: build python3.9 client - - run: | - echo "Build Python3.9 client library" - pulsar-client-cpp/docker-build-python3.9.sh \ No newline at end of file diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 266bfd57723f1..834451c1c70dd 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -40,7 +40,6 @@ jobs: runs-on: ubuntu-20.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} - cpp_only: ${{ steps.check_changes.outputs.cpp_only }} steps: - name: checkout uses: actions/checkout@v2 @@ -56,7 +55,6 @@ jobs: id: check_changes run: | echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}" - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -68,7 +66,7 @@ jobs: check-style: needs: changed_files_job - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} name: Go ${{ matrix.go-version }} Functions style check runs-on: ubuntu-20.04 strategy: diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 03e3adff33a9d..3e8a3eb46ba0c 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -41,7 +41,6 @@ jobs: runs-on: ubuntu-20.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} - cpp_only: ${{ steps.check_changes.outputs.cpp_only }} changed_tests: ${{ steps.changes.outputs.tests_files }} steps: - name: checkout @@ -58,7 +57,6 @@ jobs: id: check_changes run: | echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}" - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -73,7 +71,7 @@ jobs: name: Flaky tests suite runs-on: ubuntu-20.04 timeout-minutes: 100 - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v2 diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index e9418ee963a53..ef641feec1523 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -41,7 +41,6 @@ jobs: runs-on: ubuntu-20.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} - cpp_only: ${{ steps.check_changes.outputs.cpp_only }} changed_tests: ${{ steps.changes.outputs.tests_files }} steps: - name: checkout @@ -58,7 +57,7 @@ jobs: id: check_changes run: | echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}" + - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -141,7 +140,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['changed_files_job', 'build-and-license-check'] - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} strategy: fail-fast: false matrix: @@ -255,7 +254,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 60 needs: ['changed_files_job', 'build-and-license-check'] - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true'}} env: UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ steps: @@ -320,7 +319,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['changed_files_job', 'pulsar-java-test-image'] - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} env: PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest strategy: @@ -462,7 +461,7 @@ jobs: 'changed_files_job', 'integration-tests' ] - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v2 @@ -477,94 +476,6 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst - cpp-tests: - name: CI - CPP, Python Tests - runs-on: ubuntu-20.04 - timeout-minutes: 120 - needs: [ - 'changed_files_job', - 'integration-tests' - ] - if: always() - steps: - - name: check condition - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - run: | - if [[ ! ( ( \ - "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \ - && "${{ needs.integration-tests.result }}" == "success" \ - ) || ( \ - "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \ - ) ) ]]; then - echo "Required jobs haven't been completed successfully." - exit 1 - fi - - - name: checkout - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: actions/checkout@v2 - - - name: Tune Runner VM - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 17 - - - name: Clean Disk - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: ./.github/actions/clean-disk - - - name: Install gh-actions-artifact-client.js - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - cd $GITHUB_WORKSPACE - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution - - - name: copy python tests - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - run: | - mvn -B -Pskip-all -ntp -pl pulsar-functions/instance package - - - name: build cpp artifacts - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - run: | - echo "Build C++ client library" - pulsar-client-cpp/docker-build.sh - - - name: run c++ tests - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} - run: pulsar-client-cpp/docker-tests.sh - - - name: Upload test-logs - if: ${{ failure() && needs.changed_files_job.outputs.docs_only != 'true' }} - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - name: cpp-tests-logs - path: test-logs - pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 @@ -927,7 +838,7 @@ jobs: runs-on: macos-11 timeout-minutes: 120 needs: ['changed_files_job', 'integration-tests'] - if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} steps: - name: checkout uses: actions/checkout@v2 @@ -1033,24 +944,19 @@ jobs: 'integration-tests', 'system-tests', 'flaky-system-tests', - 'macos-build', - 'cpp-tests' + 'macos-build' ] steps: - name: Check that all required jobs were completed successfully if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} run: | if [[ ! ( ( \ - "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \ - && "${{ needs.unit-tests.result }}" == "success" \ + "${{ needs.unit-tests.result }}" == "success" \ && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ - && "${{ needs.cpp-tests.result }}" == "success" \ ) || ( \ - "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \ - && "${{ needs.system-tests.result }}" == "success" \ - && "${{ needs.cpp-tests.result }}" == "success" \ + "${{ needs.system-tests.result }}" == "success" \ ) ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 diff --git a/README.md b/README.md index 37e73f95ac194..f858e1065723e 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la ## Build Pulsar -Requirements: +### Requirements - JDK @@ -139,6 +139,7 @@ Requirements: > Note: this project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system installed Maven. > Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below. +### Build Compile and install: ```bash @@ -151,7 +152,7 @@ Compile and install individual module $ mvn -pl module-name (e.g: pulsar-broker) install -DskipTests ``` -## Minimal build (This skips most of external connectors and tiered storage handlers) +### Minimal build (This skips most of external connectors and tiered storage handlers) ``` mvn install -Pcore-modules,-main -DskipTests diff --git a/bin/pulsar b/bin/pulsar index 30fe596f60bc9..28738d84d5d86 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -348,9 +348,13 @@ ZK_OPTS=" -Dzookeeper.4lw.commands.whitelist=* -Dzookeeper.snapshot.trust.empty= LOG4J2_SHUTDOWN_HOOK_DISABLED="-Dlog4j.shutdownHookEnabled=false" -# Adding pulsar metadata as a recognized provider -BK_METADATA_OPTIONS="-Dbookkeeper.metadata.bookie.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver -Dbookkeeper.metadata.client.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver" -OPTS="$OPTS $BK_METADATA_OPTIONS" +# By default, Pulsar Metadata driver will be used for Bookkeeper client and server metadata operations +# This can be disabled by setting BK_METADATA_OPTIONS=none +if [[ "$BK_METADATA_OPTIONS" != "none" ]]; then + # Adding pulsar metadata as a recognized provider + BK_METADATA_OPTIONS="${BK_METADATA_OPTIONS:-"-Dbookkeeper.metadata.bookie.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver -Dbookkeeper.metadata.client.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"}" + OPTS="$OPTS $BK_METADATA_OPTIONS" +fi #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index 71a816a9b2b73..ab69544308ff3 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -120,8 +120,8 @@ test_group_messaging() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests # run integration proxy tests mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests - # run integration proxy with WebSocket tests - mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests + # run integration WebSocket tests + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-websocket.xml -DintegrationTests } test_group_plugin() { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java index 73fff1bb7e269..ff590c05bbcf7 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java @@ -38,11 +38,15 @@ public class MockitoCleanupListener extends BetweenTestClassesListenerAdapter { @Override protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { if (MOCKITO_CLEANUP_ENABLED) { - if (MockitoThreadLocalStateCleaner.INSTANCE.isEnabled()) { - LOG.info("Cleaning up Mockito's ThreadSafeMockingProgress.MOCKING_PROGRESS_PROVIDER thread local state."); - MockitoThreadLocalStateCleaner.INSTANCE.cleanup(); + try { + if (MockitoThreadLocalStateCleaner.INSTANCE.isEnabled()) { + LOG.info( + "Cleaning up Mockito's ThreadSafeMockingProgress.MOCKING_PROGRESS_PROVIDER thread local state."); + MockitoThreadLocalStateCleaner.INSTANCE.cleanup(); + } + } finally { + cleanupMockitoInline(); } - cleanupMockitoInline(); } } @@ -54,5 +58,4 @@ protected void onBetweenTestClasses(Class endedTestClass, Class startedTes private void cleanupMockitoInline() { Mockito.framework().clearInlineMocks(); } - } diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoThreadLocalStateCleaner.java b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoThreadLocalStateCleaner.java index 3c383fb1d92fd..4fe4ea7e0e2d3 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoThreadLocalStateCleaner.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoThreadLocalStateCleaner.java @@ -23,6 +23,8 @@ import org.apache.commons.lang3.ClassUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.reflect.MethodUtils; +import org.mockito.internal.stubbing.InvocationContainerImpl; +import org.mockito.internal.util.MockUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,16 +74,33 @@ public void cleanup() { LOG.warn("Invalid usage of Mockito detected on thread {}." + " There is ongoing stubbing on mock of class={} instance={}", thread, mock.getClass().getName(), mock); + try { + clearInvocations(thread, mock); + } catch (Exception e) { + LOG.warn("Clearing invocations failed", e); + } } } } catch (NoSuchMethodException | IllegalAccessException e) { LOG.debug("Cannot call validateState on existing Mockito ProgressProvider"); } catch (InvocationTargetException e) { LOG.warn("Invalid usage of Mockito detected on thread {}", thread, e.getCause()); + } catch (Exception e) { + LOG.warn("Removing {} instance from thread {} failed", mockingProgress.getClass().getName(), thread, e); } }); } + private static void clearInvocations(Thread thread, Object mock) { + InvocationContainerImpl invocationContainer = MockUtil.getInvocationContainer(mock); + if (invocationContainer.hasInvocationForPotentialStubbing()) { + LOG.warn("Mock contains registered invocations that should be cleared. thread {} class={} " + + "instance={}", + thread, mock.getClass().getName(), mock); + invocationContainer.clearInvocations(); + } + } + public boolean isEnabled() { return MOCKING_PROGRESS_PROVIDER != null; } diff --git a/conf/broker.conf b/conf/broker.conf index 30e79ebc9f090..e6b3aef8811b5 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -453,6 +453,12 @@ entryFilterNames= # The directory for all the entry filter implementations entryFiltersDirectory= +# Whether the broker should count filtered entries in dispatch rate limit calculations. When disabled, +# only messages sent to a consumer count towards a dispatch rate limit at the broker, topic, and +# subscription level. When enabled, messages filtered out due to entry filter logic are counted towards +# each relevant rate limit. +dispatchThrottlingForFilteredEntriesEnabled=false + # Whether allow topic level entry filters policies overrides broker configuration. allowOverrideEntryFilters=false @@ -570,6 +576,12 @@ delayedDeliveryMaxNumBuckets=50 # Enable share the delayed message index across subscriptions delayedDeliverySharedIndexEnabled=false +# Size of the lookahead window to use when detecting if all the messages in the topic +# have a fixed delay. +# Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle +# fixed delays in messages in a different way. +delayedDeliveryFixedDelayDetectionLookahead=50000 + # Whether to enable acknowledge of batch local index. acknowledgmentAtBatchIndexLevelEnabled=false diff --git a/conf/client.conf b/conf/client.conf index ea1d339a09c5b..8a485e5676c7b 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -88,7 +88,11 @@ tlsKeyStorePassword= # When TLS authentication with KeyStore is used, available options can be SunJSSE, Conscrypt and so on. webserviceTlsProvider= +#Proxy-server URL to which to connect +proxyServiceUrl= +#Proxy protocol to select type of routing at proxy +proxyProtocol= # Pulsar Admin Custom Commands #customCommandFactoriesDirectory=commandFactories diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 3746fe1c26c2e..7ec8788b89c60 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -89,13 +89,14 @@ RUN pip3 install pyyaml==5.4.1 # 4. /pulsar - hadoop writes to this directory RUN mkdir /pulsar && chmod g+w /pulsar -ADD target/python-client/ /pulsar/pulsar-client - ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar +ARG PULSAR_CLIENT_PYTHON_VERSION +ENV PULSAR_CLIENT_PYTHON_VERSION ${PULSAR_CLIENT_PYTHON_VERSION} + # This script is intentionally run as the root user to make the dependencies available for all UIDs. RUN /pulsar/bin/install-pulsar-client.sh diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index a458cea39763a..f7d91b2120841 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -30,12 +30,6 @@ Apache Pulsar :: Docker Images :: Pulsar Latest Version pom - - x86_64 - false - false - - ${project.groupId} @@ -58,55 +52,11 @@ docker target/pulsar-server-distribution-${project.version}-bin.tar.gz + ${pulsar.client.python.version} ${env.UBUNTU_MIRROR} - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - build-pulsar-clients-python-35 - compile - - exec - - - ${skipBuildPythonClient} - ${project.basedir}/target - ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - - - 3.8 cp38-cp38 manylinux2014 ${pythonClientBuildArch} - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - copy-pulsar-clients-python - compile - - run - - - ${skipCopyPythonClients} - - copy python wheel file - - - - - - - io.fabric8 docker-maven-plugin diff --git a/docker/pulsar/scripts/install-pulsar-client.sh b/docker/pulsar/scripts/install-pulsar-client.sh index 355f5f6524a3a..0951b2aec1b60 100755 --- a/docker/pulsar/scripts/install-pulsar-client.sh +++ b/docker/pulsar/scripts/install-pulsar-client.sh @@ -27,6 +27,4 @@ if [ "${ARCH}" == "arm64" ]; then apt -y install build-essential python3-dev fi -PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/') -WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}") -pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all] +pip3 install pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 17dbac09a2292..cf022d3552e64 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -18,13 +18,13 @@ */ package org.apache.bookkeeper.mledger; -import com.google.common.base.Predicate; import com.google.common.collect.Range; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index a71cf38afb855..c5de804b1379d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -20,6 +20,7 @@ import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; import org.apache.bookkeeper.common.annotation.InterfaceAudience; @@ -649,6 +650,12 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro */ CompletableFuture getLedgerInfo(long ledgerId); + /** + * Get basic ledger summary. + * will get {@link Optional#empty()} if corresponding ledger not exists. + */ + Optional getOptionalLedgerInfo(long ledgerId); + /** * Truncate ledgers * The truncate operation will move all cursors to the end of the topic and delete all inactive ledgers. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 92c9c91198134..0efd1ca2a823c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -32,6 +32,7 @@ import org.apache.bookkeeper.common.annotation.InterfaceStability; import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; +import org.apache.commons.collections4.MapUtils; import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; /** @@ -742,4 +743,10 @@ public int getMaxBacklogBetweenCursorsForCaching() { public void setMaxBacklogBetweenCursorsForCaching(int maxBacklogBetweenCursorsForCaching) { this.maxBacklogBetweenCursorsForCaching = maxBacklogBetweenCursorsForCaching; } + + public String getShadowSource() { + return MapUtils.getString(properties, PROPERTY_SOURCE_TOPIC_KEY); + } + + public static final String PROPERTY_SOURCE_TOPIC_KEY = "PULSAR.SHADOW_SOURCE"; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java index 0dc820ec46d72..fa6dc59d14753 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java @@ -147,6 +147,12 @@ public NonRecoverableLedgerException(String msg) { } } + public static class LedgerNotExistException extends NonRecoverableLedgerException { + public LedgerNotExistException(String msg) { + super(msg); + } + } + public static class InvalidReplayPositionException extends ManagedLedgerException { public InvalidReplayPositionException(String msg) { super(msg); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java index e42c2581ba101..667d641ac9ae0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import org.apache.bookkeeper.common.annotation.InterfaceAudience; @@ -197,4 +198,8 @@ void asyncOpenReadOnlyCursor(String managedLedgerName, Position startPosition, M * */ long getCacheEvictionTimeThreshold(); + /** + * @return properties of this managedLedger. + */ + CompletableFuture> getManagedLedgerPropertiesAsync(String name); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java index 28454d647b008..fe086f009808e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java @@ -18,9 +18,9 @@ */ package org.apache.bookkeeper.mledger; -import com.google.common.base.Predicate; import com.google.common.collect.Range; import java.util.List; +import java.util.function.Predicate; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java index b5a5be733a136..293ce8d2fcbc2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java @@ -286,7 +286,7 @@ public void remove() { // ////////////////////// /** - * Push the item up towards the the root of the tree (lowest reading position). + * Push the item up towards the root of the tree (the lowest reading position). */ private void siftUp(Item item) { Item parent = getParent(item); @@ -297,7 +297,7 @@ private void siftUp(Item item) { } /** - * Push the item down towards the bottom of the tree (highest reading position). + * Push the item down towards the bottom of the tree (the highest reading position). */ private void siftDown(final Item item) { while (true) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 6d595e76dc127..b67c74fdc7956 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -28,7 +28,6 @@ import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.base.Predicate; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Range; @@ -60,6 +59,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; +import java.util.function.Predicate; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; import org.apache.bookkeeper.client.BKException; @@ -260,11 +260,11 @@ public void triggerFailed(ManagedLedgerException exception) { AtomicIntegerFieldUpdater.newUpdater(ManagedCursorImpl.class, "pendingMarkDeletedSubmittedCount"); @SuppressWarnings("unused") private volatile int pendingMarkDeletedSubmittedCount = 0; - private long lastLedgerSwitchTimestamp; + private volatile long lastLedgerSwitchTimestamp; private final Clock clock; // The last active time (Unix time, milliseconds) of the cursor - private long lastActive; + private volatile long lastActive; public enum State { Uninitialized, // Cursor is being initialized diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index d7596a7468a40..e0af1cc632612 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -367,11 +367,13 @@ public void asyncOpen(final String name, final ManagedLedgerConfig config, final ledgers.computeIfAbsent(name, (mlName) -> { // Create the managed ledger CompletableFuture future = new CompletableFuture<>(); - final ManagedLedgerImpl newledger = new ManagedLedgerImpl(this, - bookkeeperFactory.get( - new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), - config.getBookKeeperEnsemblePlacementPolicyProperties())), - store, config, scheduledExecutor, name, mlOwnershipChecker); + BookKeeper bk = bookkeeperFactory.get( + new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), + config.getBookKeeperEnsemblePlacementPolicyProperties())); + final ManagedLedgerImpl newledger = config.getShadowSource() == null + ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, mlOwnershipChecker) + : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, + mlOwnershipChecker); PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger); pendingInitializeLedgers.put(name, pendingLedger); newledger.initialize(new ManagedLedgerInitializeLedgerCallback() { @@ -954,6 +956,11 @@ public void operationFailed(MetaStoreException e) { return future; } + @Override + public CompletableFuture> getManagedLedgerPropertiesAsync(String name) { + return store.getManagedLedgerPropertiesAsync(name); + } + public MetaStore getMetaStore() { return store; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 71bc8cad6f0b3..10de447d0f49e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -100,6 +100,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.InvalidCursorPositionException; +import org.apache.bookkeeper.mledger.ManagedLedgerException.LedgerNotExistException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerInterceptException; @@ -215,13 +216,13 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { private volatile LedgerHandle currentLedger; private long currentLedgerEntries = 0; private long currentLedgerSize = 0; - private long lastLedgerCreatedTimestamp = 0; - private long lastLedgerCreationFailureTimestamp = 0; + private volatile long lastLedgerCreatedTimestamp = 0; + private volatile long lastLedgerCreationFailureTimestamp = 0; private long lastLedgerCreationInitiationTimestamp = 0; private long lastOffloadLedgerId = 0; - private long lastOffloadSuccessTimestamp = 0; - private long lastOffloadFailureTimestamp = 0; + private volatile long lastOffloadSuccessTimestamp = 0; + private volatile long lastOffloadFailureTimestamp = 0; private int minBacklogCursorsForCaching = 0; private int minBacklogEntriesForCaching = 1000; @@ -458,9 +459,7 @@ private synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedg } else { iterator.remove(); bookKeeper.asyncDeleteLedger(li.getLedgerId(), (rc, ctx) -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc); - } + log.info("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc); }, null); } } @@ -1837,6 +1836,11 @@ public CompletableFuture getLedgerInfo(long ledgerId) { return result; } + @Override + public Optional getOptionalLedgerInfo(long ledgerId) { + return Optional.ofNullable(ledgers.get(ledgerId)); + } + CompletableFuture getLedgerHandle(long ledgerId) { CompletableFuture ledgerHandle = ledgerCache.get(ledgerId); if (ledgerHandle != null) { @@ -1943,7 +1947,7 @@ public void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Ob } else { log.error("[{}] Failed to get message with ledger {}:{} the ledgerId does not belong to this topic " + "or has been deleted.", name, position.getLedgerId(), position.getEntryId()); - callback.readEntryFailed(new ManagedLedgerException.NonRecoverableLedgerException("Message not found, " + callback.readEntryFailed(new LedgerNotExistException("Message not found, " + "the ledgerId does not belong to this topic or has been deleted"), ctx); } @@ -3475,7 +3479,7 @@ public PositionImpl getNextValidPositionInternal(final PositionImpl position) { while (!isValidPosition(nextPosition)) { Long nextLedgerId = ledgers.ceilingKey(nextPosition.getLedgerId() + 1); if (nextLedgerId == null) { - throw new NullPointerException(); + throw new NullPointerException("nextLedgerId is null. No valid next position after " + position); } nextPosition = PositionImpl.get(nextLedgerId, 0); } @@ -3756,11 +3760,26 @@ private static boolean isBkErrorNotRecoverable(int rc) { } } + private static boolean isLedgerNotExistException(int rc) { + switch (rc) { + case Code.NoSuchLedgerExistsException: + case Code.NoSuchLedgerExistsOnMetadataServerException: + return true; + + default: + return false; + } + } + public static ManagedLedgerException createManagedLedgerException(int bkErrorCode) { if (bkErrorCode == BKException.Code.TooManyRequestsException) { return new TooManyRequestsException("Too many request error from bookies"); } else if (isBkErrorNotRecoverable(bkErrorCode)) { - return new NonRecoverableLedgerException(BKException.getMessage(bkErrorCode)); + if (isLedgerNotExistException(bkErrorCode)) { + return new LedgerNotExistException(BKException.getMessage(bkErrorCode)); + } else { + return new NonRecoverableLedgerException(BKException.getMessage(bkErrorCode)); + } } else { return new ManagedLedgerException(BKException.getMessage(bkErrorCode)); } @@ -4118,6 +4137,7 @@ public CompletableFuture getManagedLedgerInternalSta cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); cs.state = cursor.getState(); + cs.active = cursor.isActive(); cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage(); cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange(); cs.properties = cursor.getProperties(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java index 35f109b21dc5c..3d60066782af2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java @@ -158,4 +158,13 @@ void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedCursorIn * if the operation succeeds. */ CompletableFuture asyncExists(String ledgerName); + + + /** + * Get managed ledger properties from meta store. + * + * @param name ledgerName + * @return a future represents the result of the operation. + */ + CompletableFuture> getManagedLedgerPropertiesAsync(String name); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index a501b9e43dc0f..2949902ac353f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -23,6 +23,8 @@ import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -148,6 +150,33 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map }); } + public CompletableFuture> getManagedLedgerPropertiesAsync(String name) { + CompletableFuture> result = new CompletableFuture<>(); + getManagedLedgerInfo(name, false, new MetaStoreCallback<>() { + @Override + public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) { + HashMap propertiesMap = new HashMap<>(mlInfo.getPropertiesCount()); + if (mlInfo.getPropertiesCount() > 0) { + for (int i = 0; i < mlInfo.getPropertiesCount(); i++) { + MLDataFormats.KeyValue property = mlInfo.getProperties(i); + propertiesMap.put(property.getKey(), property.getValue()); + } + } + result.complete(propertiesMap); + } + + @Override + public void operationFailed(MetaStoreException e) { + if (e instanceof MetadataNotFoundException) { + result.complete(Collections.emptyMap()); + } else { + result.completeExceptionally(e); + } + } + }); + return result; + } + @Override public void asyncUpdateLedgerIds(String ledgerName, ManagedLedgerInfo mlInfo, Stat stat, MetaStoreCallback callback) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java index e65e418a5ec82..9e56711b38667 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java @@ -18,11 +18,11 @@ */ package org.apache.bookkeeper.mledger.impl; -import com.google.common.base.Predicate; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Predicate; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.ScanCallback; @@ -81,7 +81,7 @@ public void readEntriesComplete(List entries, Object ctx) { callback.scanComplete(lastSeenPosition, ScanOutcome.ABORTED, OpScan.this.ctx); return; } - if (!condition.apply(entry)) { + if (!condition.test(entry)) { log.warn("[{}] Scan abort due to user code", OpScan.this.cursor); callback.scanComplete(lastSeenPosition, ScanOutcome.USER_INTERRUPTED, OpScan.this.ctx); return; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java new file mode 100644 index 0000000000000..346780a228349 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.bookkeeper.mledger.impl; + +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.pulsar.common.naming.TopicName; + +/** + * Working in progress until PIP-180 is finished. + * Currently, it works nothing different with ManagedLedgerImpl. + */ +@Slf4j +public class ShadowManagedLedgerImpl extends ManagedLedgerImpl { + + private final TopicName shadowSource; + private final String sourceMLName; + + public ShadowManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, + MetaStore store, ManagedLedgerConfig config, + OrderedScheduler scheduledExecutor, + String name, final Supplier mlOwnershipChecker) { + super(factory, bookKeeper, store, config, scheduledExecutor, name, mlOwnershipChecker); + this.shadowSource = TopicName.get(config.getShadowSource()); + this.sourceMLName = shadowSource.getPersistenceNamingEncoding(); + } + + @Override + synchronized void initialize(ManagedLedgerInitializeLedgerCallback callback, Object ctx) { + // TODO: ShadowManagedLedger has different initialize process from normal ManagedLedger, + // which is complicated and will be implemented in the next PRs. + super.initialize(callback, ctx); + } + + public TopicName getShadowSource() { + return shadowSource; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java index 81c89b37d5543..640a7eeefcebb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java @@ -39,7 +39,7 @@ public interface EntryCache extends Comparable { /** * Insert an entry in the cache. * - *

If the overall limit have been reached, this will triggered the eviction of other entries, possibly from + *

If the overall limit have been reached, this will trigger the eviction of other entries, possibly from * other EntryCache instances * * @param entry @@ -49,7 +49,7 @@ public interface EntryCache extends Comparable { boolean insert(EntryImpl entry); /** - * Remove from cache all the entries related to a ledger up to lastPosition included. + * Remove from cache all the entries related to a ledger up to lastPosition excluded. * * @param lastPosition * the position of the last entry to be invalidated (non-inclusive) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index 312e09f846eb1..dfb65d59c78bd 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -25,7 +25,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.base.Predicate; import com.google.common.collect.Lists; import com.google.common.collect.Range; import com.google.common.collect.Sets; @@ -34,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java index 39f9dc9ba7d84..e4f1d470bf6de 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java @@ -184,7 +184,7 @@ protected String changeLedgerPath() { return ""; } - @AfterTest + @AfterTest(alwaysRun = true) public void tearDown() throws Exception { boolean failed = false; for (Throwable e : asyncExceptions) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java index f0dcd1d788355..6f9d04d9eb0b3 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java @@ -144,8 +144,8 @@ public void restartCluster() throws Exception { if (0 == zooKeeperPort) { zooKeeperPort = serverFactory.getLocalPort(); - zkaddr = new InetSocketAddress(zkaddr.getHostName(), zooKeeperPort); - connectString = zkaddr.getHostName() + ":" + zooKeeperPort; + zkaddr = new InetSocketAddress(zkaddr.getAddress().getHostAddress(), zooKeeperPort); + connectString = zkaddr.getAddress().getHostAddress() + ":" + zooKeeperPort; } boolean b = ClientBase.waitForServerUp(getZooKeeperConnectString(), diff --git a/pom.xml b/pom.xml index 8bd36cc63d263..a32dd8f377aac 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,8 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 + 2.10.1 + **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java @@ -252,7 +254,7 @@ flexible messaging model and an intuitive client API. 3.0.0 4.0.rc2 1.0 - 3.0.0 + 3.1.0 @@ -279,6 +281,7 @@ flexible messaging model and an intuitive client API. 0.4 7.1.0 0.9.15 + 1.6.1 rename-netty-native-libs.sh diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index a6e9a556820a6..6683d36c36e06 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -372,6 +372,12 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se @FieldContext(category = CATEGORY_SERVER, doc = "Enable share the delayed message index across subscriptions") private boolean delayedDeliverySharedIndexEnabled = false; + @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use " + + "when detecting if all the messages in the topic have a fixed delay. " + + "Default is 50,000. Setting the lookahead window to 0 will disable the " + + "logic to handle fixed delays in messages in a different way.") + private long delayedDeliveryFixedDelayDetectionLookahead = 50_000; + @FieldContext(category = CATEGORY_SERVER, doc = "Whether to enable the acknowledge of batch local index") private boolean acknowledgmentAtBatchIndexLevelEnabled = false; @@ -1050,6 +1056,16 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private boolean dispatcherDispatchMessagesInSubscriptionThread = true; + @FieldContext( + dynamic = false, + category = CATEGORY_SERVER, + doc = "Whether the broker should count filtered entries in dispatch rate limit calculations. When disabled, " + + "only messages sent to a consumer count towards a dispatch rate limit at the broker, topic, and " + + "subscription level. When enabled, messages filtered out due to entry filter logic are counted towards " + + "each relevant rate limit." + ) + private boolean dispatchThrottlingForFilteredEntriesEnabled = false; + // <-- dispatcher read settings --> @FieldContext( dynamic = true, diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java index d341e699bf7e8..c2aa40e030d80 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.util.Enumeration; import java.util.List; +import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.common.util.SimpleTextOutputStream; /** @@ -65,16 +66,22 @@ public static void generateSystemMetrics(SimpleTextOutputStream stream, String c for (int i = 0; i < metricFamily.samples.size(); i++) { Collector.MetricFamilySamples.Sample sample = metricFamily.samples.get(i); stream.write(sample.name); + stream.write("{"); if (!sample.labelNames.contains("cluster")) { - stream.write("{cluster=\"").write(cluster).write('"'); + stream.write("cluster=\"").write(cluster).write('"'); + // If label is empty, should not append ','. + if (!CollectionUtils.isEmpty(sample.labelNames)){ + stream.write(","); + } } for (int j = 0; j < sample.labelNames.size(); j++) { String labelValue = sample.labelValues.get(j); if (labelValue != null) { labelValue = labelValue.replace("\"", "\\\""); } - - stream.write(","); + if (j > 0) { + stream.write(","); + } stream.write(sample.labelNames.get(j)); stream.write("=\""); stream.write(labelValue); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java new file mode 100644 index 0000000000000..9bbfa5d771438 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.testng.Assert.*; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import java.io.ByteArrayOutputStream; +import java.util.Collections; +import java.util.UUID; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class PrometheusMetricsGeneratorUtilsTest { + + private static final String LABEL_NAME_CLUSTER = "cluster"; + + @Test + public void testGenerateSystemMetricsWithSpecifyCluster() throws Exception { + String defaultClusterValue = "cluster_test"; + String specifyClusterValue = "lb_x"; + String metricsName = "label_contains_cluster" + randomString(); + Counter counter = new Counter.Builder() + .name(metricsName) + .labelNames(LABEL_NAME_CLUSTER) + .help("x") + .register(CollectorRegistry.defaultRegistry); + counter.labels(specifyClusterValue).inc(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); + assertTrue(out.toString().contains( + String.format("%s_total{cluster=\"%s\"} 1.0", metricsName, specifyClusterValue) + )); + // cleanup + out.close(); + CollectorRegistry.defaultRegistry.unregister(counter); + } + + @Test + public void testGenerateSystemMetricsWithDefaultCluster() throws Exception { + String defaultClusterValue = "cluster_test"; + String labelName = "lb_name"; + String labelValue = "lb_value"; + // default cluster. + String metricsName = "label_use_default_cluster" + randomString(); + Counter counter = new Counter.Builder() + .name(metricsName) + .labelNames(labelName) + .help("x") + .register(CollectorRegistry.defaultRegistry); + counter.labels(labelValue).inc(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); + assertTrue(out.toString().contains( + String.format("%s_total{cluster=\"%s\",%s=\"%s\"} 1.0", + metricsName, defaultClusterValue, labelName, labelValue) + )); + // cleanup + out.close(); + CollectorRegistry.defaultRegistry.unregister(counter); + } + + @Test + public void testGenerateSystemMetricsWithoutCustomizedLabel() throws Exception { + String defaultClusterValue = "cluster_test"; + // default cluster. + String metricsName = "label_without_customized_label" + randomString(); + Counter counter = new Counter.Builder() + .name(metricsName) + .help("x") + .register(CollectorRegistry.defaultRegistry); + counter.inc(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); + assertTrue(out.toString().contains( + String.format("%s_total{cluster=\"%s\"} 1.0", metricsName, defaultClusterValue) + )); + // cleanup + out.close(); + CollectorRegistry.defaultRegistry.unregister(counter); + } + + private static String randomString(){ + return UUID.randomUUID().toString().replaceAll("-", ""); + } +} diff --git a/pulsar-client-cpp/include/pulsar/c/version.h b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/package-info.java similarity index 94% rename from pulsar-client-cpp/include/pulsar/c/version.h rename to pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/package-info.java index ab63c8a708991..3723fb4ff5c99 100644 --- a/pulsar-client-cpp/include/pulsar/c/version.h +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/package-info.java @@ -16,7 +16,4 @@ * specific language governing permissions and limitations * under the License. */ - -#pragma once - -#include +package org.apache.pulsar.broker.stats.prometheus; diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index df04da60bc615..ca36f9ca3710d 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -437,10 +437,6 @@ true 17 - org.apache.pulsar.broker.admin - org.apache.pulsar.broker.namespace - org.apache.pulsar.broker.service - org.apache.pulsar.broker.stats org.apache.pulsar.client diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index a95e6121b3e53..1b4edf8686eac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -35,6 +35,7 @@ import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.client.api.ProxyProtocol; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; @@ -63,10 +64,18 @@ */ public class PulsarClusterMetadataSetup { + private static final int DEFAULT_BUNDLE_NUMBER = 16; + private static class Arguments { @Parameter(names = { "-c", "--cluster" }, description = "Cluster name", required = true) private String cluster; + @Parameter(names = {"-bn", + "--default-namespace-bundle-number"}, + description = "The bundle numbers for the default namespaces(public/default), default is 16", + required = false) + private int numberOfDefaultNamespaceBundles; + @Parameter(names = { "-uw", "--web-service-url" }, description = "Web-service URL for new cluster", required = true) private String clusterWebServiceUrl; @@ -139,6 +148,16 @@ private static class Arguments { hidden = true) private String bookieMetadataServiceUri; + @Parameter(names = { "-pp", + "--proxy-protocol" }, + description = "Proxy protocol to select type of routing at proxy. Possible Values: [SNI]", + required = false) + private ProxyProtocol clusterProxyProtocol; + + @Parameter(names = { "-pu", + "--proxy-url" }, description = "Proxy-server URL to which to connect.", required = false) + private String clusterProxyUrl; + @Parameter(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; @@ -232,9 +251,11 @@ public static void main(String[] args) throws Exception { System.err.println("Number of transaction coordinators must greater than 0"); System.exit(1); } - + int bundleNumberForDefaultNamespace = + arguments.numberOfDefaultNamespaceBundles > 0 ? arguments.numberOfDefaultNamespaceBundles + : DEFAULT_BUNDLE_NUMBER; try { - initializeCluster(arguments); + initializeCluster(arguments, bundleNumberForDefaultNamespace); } catch (Exception e) { System.err.println("Unexpected error occured."); e.printStackTrace(System.err); @@ -243,7 +264,7 @@ public static void main(String[] args) throws Exception { } } - private static void initializeCluster(Arguments arguments) throws Exception { + private static void initializeCluster(Arguments arguments, int bundleNumberForDefaultNamespace) throws Exception { log.info("Setting up cluster {} with metadata-store={} configuration-metadata-store={}", arguments.cluster, arguments.metadataStoreUrl, arguments.configurationMetadataStore); @@ -299,6 +320,8 @@ private static void initializeCluster(Arguments arguments) throws Exception { .serviceUrlTls(arguments.clusterWebServiceUrlTls) .brokerServiceUrl(arguments.clusterBrokerServiceUrl) .brokerServiceUrlTls(arguments.clusterBrokerServiceUrlTls) + .proxyServiceUrl(arguments.clusterProxyUrl) + .proxyProtocol(arguments.clusterProxyProtocol) .build(); if (!resources.getClusterResources().clusterExists(arguments.cluster)) { resources.getClusterResources().createCluster(arguments.cluster, clusterData); @@ -318,7 +341,7 @@ private static void initializeCluster(Arguments arguments) throws Exception { // Create default namespace createNamespaceIfAbsent(resources, NamespaceName.get(TopicName.PUBLIC_TENANT, TopicName.DEFAULT_NAMESPACE), - arguments.cluster); + arguments.cluster, bundleNumberForDefaultNamespace); // Create system namespace createNamespaceIfAbsent(resources, NamespaceName.SYSTEM_NAMESPACE, arguments.cluster); @@ -350,17 +373,18 @@ static void createTenantIfAbsent(PulsarResources resources, String tenant, Strin } } - static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, String cluster) - throws IOException { + static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, + String cluster, int bundleNumber) throws IOException { NamespaceResources namespaceResources = resources.getNamespaceResources(); if (!namespaceResources.namespaceExists(namespaceName)) { Policies policies = new Policies(); - policies.bundles = getBundles(16); + policies.bundles = getBundles(bundleNumber); policies.replication_clusters = Collections.singleton(cluster); namespaceResources.createPolicies(namespaceName, policies); } else { + log.info("Namespace {} already exists.", namespaceName); namespaceResources.setPolicies(namespaceName, policies -> { policies.replication_clusters.add(cluster); return policies; @@ -368,6 +392,11 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam } } + static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, + String cluster) throws IOException { + createNamespaceIfAbsent(resources, namespaceName, cluster, DEFAULT_BUNDLE_NUMBER); + } + static void createPartitionedTopic(MetadataStore configStore, TopicName topicName, int numPartitions) throws InterruptedException, IOException, ExecutionException { PulsarResources resources = new PulsarResources(null, configStore); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index c69c6e2418f7c..0ade26e661254 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -112,6 +112,7 @@ import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider; import org.apache.pulsar.broker.validator.MultipleListenerValidator; import org.apache.pulsar.broker.validator.TransactionBatchedWriteValidator; import org.apache.pulsar.broker.web.WebService; @@ -168,6 +169,7 @@ import org.apache.pulsar.packages.management.core.impl.PackagesManagementImpl; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreProvider; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; @@ -419,6 +421,14 @@ public CompletableFuture closeAsync() { } this.resourceUsageTransportManager = null; } + if (this.resourceGroupServiceManager != null) { + try { + this.resourceGroupServiceManager.close(); + } catch (Exception e) { + LOG.warn("ResourceGroupServiceManager closing failed {}", e.getMessage()); + } + this.resourceGroupServiceManager = null; + } if (this.webService != null) { try { @@ -822,6 +832,9 @@ public void start() throws PulsarServerException { // Register pulsar system namespaces and start transaction meta store service if (config.isTransactionCoordinatorEnabled()) { + MLTransactionMetadataStoreProvider.initBufferedWriterMetrics(getAdvertisedAddress()); + MLPendingAckStoreProvider.initBufferedWriterMetrics(getAdvertisedAddress()); + this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient()); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 47c69ecc69846..bab7ed0542402 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -802,7 +803,7 @@ protected CompletableFuture getSchemaCompatibilityS @CanIgnoreReturnValue public static T checkNotNull(T reference) { - return com.google.common.base.Preconditions.checkNotNull(reference); + return Objects.requireNonNull(reference); } protected void checkNotNull(Object o, String errorMessage) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 93ddbf794c0d2..01532449687ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -22,7 +22,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.common.policies.data.PoliciesUtil.defaultBundle; import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.net.MalformedURLException; @@ -176,7 +175,7 @@ protected CompletableFuture> internalGetListOfTopics(Policies polic } protected CompletableFuture> internalGetNonPersistentTopics(Policies policies) { - final List>> futures = Lists.newArrayList(); + final List>> futures = new ArrayList<>(); final List boundaries = policies.bundles.getBoundaries(); for (int i = 0; i < boundaries.size() - 1; i++) { final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); @@ -189,7 +188,7 @@ protected CompletableFuture> internalGetNonPersistentTopics(Policie } return FutureUtil.waitForAll(futures) .thenApply(__ -> { - final List topics = Lists.newArrayList(); + final List topics = new ArrayList<>(); for (int i = 0; i < futures.size(); i++) { List topicList = futures.get(i).join(); if (topicList != null) { @@ -341,7 +340,7 @@ private CompletableFuture precheckWhenDeleteNamespace(NamespaceName ns if (replicationClusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) { // the only replication cluster is other cluster, redirect - String replCluster = Lists.newArrayList(policies.replication_clusters).get(0); + String replCluster = new ArrayList<>(policies.replication_clusters).get(0); return clusterResources().getClusterAsync(replCluster) .thenCompose(replClusterDataOpt -> { ClusterData replClusterData = replClusterDataOpt @@ -423,7 +422,7 @@ protected CompletableFuture internalDeleteNamespaceBundleAsync(String bund if (policies.replication_clusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) { // the only replication cluster is other cluster, redirect - String replCluster = Lists.newArrayList(policies.replication_clusters).get(0); + String replCluster = new ArrayList<>(policies.replication_clusters).get(0); future = clusterResources().getClusterAsync(replCluster) .thenCompose(clusterData -> { if (clusterData.isEmpty()) { @@ -768,7 +767,7 @@ protected void internalUnloadNamespace(AsyncResponse asyncResponse) { Policies policies = getNamespacePolicies(namespaceName); - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); List boundaries = policies.bundles.getBoundaries(); for (int i = 0; i < boundaries.size() - 1; i++) { String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); @@ -814,7 +813,7 @@ protected CompletableFuture internalUnloadNamespaceAsync() { }) .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) .thenCompose(policies -> { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); List boundaries = policies.bundles.getBoundaries(); for (int i = 0; i < boundaries.size() - 1; i++) { String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); @@ -1342,7 +1341,7 @@ private CompletableFuture doUpdatePersistenceAsync(PersistencePolicies per protected void internalClearNamespaceBacklog(AsyncResponse asyncResponse, boolean authoritative) { validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); try { NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName); @@ -1407,7 +1406,7 @@ protected void internalClearNamespaceBacklogForSubscription(AsyncResponse asyncR validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); checkNotNull(subscription, "Subscription should not be null"); - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); try { NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName); @@ -1474,7 +1473,7 @@ protected void internalUnsubscribeNamespace(AsyncResponse asyncResponse, String validateNamespaceOperation(namespaceName, NamespaceOperation.UNSUBSCRIBE); checkNotNull(subscription, "Subscription should not be null"); - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); try { NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName); @@ -1721,7 +1720,7 @@ private void clearBacklog(NamespaceName nsName, String bundleRange, String subsc List topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange); - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); if (subscription != null) { if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { subscription = PersistentReplicator.getRemoteCluster(subscription); @@ -1753,7 +1752,7 @@ private void unsubscribe(NamespaceName nsName, String bundleRange, String subscr try { List topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange); - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { throw new RestException(Status.PRECONDITION_FAILED, "Cannot unsubscribe a replication cursor"); } else { @@ -1794,7 +1793,7 @@ protected BundlesData validateBundlesData(BundlesData initialBundles) { throw new RestException(Status.BAD_REQUEST, "Input bundles do not cover the whole hash range. first:" + partitions.first() + ", last:" + partitions.last()); } - List bundles = Lists.newArrayList(); + List bundles = new ArrayList<>(); bundles.addAll(partitions); return BundlesData.builder() .boundaries(bundles) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 04a4ee7a23e4e..86b242c949760 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -22,8 +22,6 @@ import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.zafarkhaja.semver.Version; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import java.io.IOException; @@ -251,7 +249,7 @@ protected CompletableFuture>> internalGetPermissions throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); } - Map> permissions = Maps.newHashMap(); + Map> permissions = new HashMap<>(); String topicUri = topicName.toString(); AuthPolicies auth = policies.get().auth_policies; // First add namespace level permissions @@ -871,8 +869,7 @@ protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authorit getPartitionedTopicMetadataAsync(topicName, authoritative, false) .thenAccept(meta -> { if (meta.partitions > 0) { - final List> futures = - Lists.newArrayListWithCapacity(meta.partitions); + final List> futures = new ArrayList<>(meta.partitions); for (int i = 0; i < meta.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); try { @@ -1145,7 +1142,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut final Set subscriptions = Collections.newSetFromMap( new ConcurrentHashMap<>(partitionMetadata.partitions)); - final List> subscriptionFutures = Lists.newArrayList(); + final List> subscriptionFutures = new ArrayList<>(); if (topicName.getDomain() == TopicDomain.persistent) { final Map> existsFutures = new ConcurrentHashMap<>(partitionMetadata.partitions); @@ -1153,7 +1150,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut existsFutures.put(i, topicResources().persistentTopicExists(topicName.getPartition(i))); } - FutureUtil.waitForAll(Lists.newArrayList(existsFutures.values())) + FutureUtil.waitForAll(new ArrayList<>(existsFutures.values())) .thenApply(unused2 -> existsFutures.entrySet().stream().filter(e -> e.getValue().join()) .map(item -> topicName.getPartition(item.getKey()).toString()) @@ -1243,7 +1240,7 @@ private void resumeAsyncResponse(AsyncResponse asyncResponse, Set subscr private void internalGetSubscriptionsForNonPartitionedTopic(AsyncResponse asyncResponse) { getTopicReferenceAsync(topicName) - .thenAccept(topic -> asyncResponse.resume(Lists.newArrayList(topic.getSubscriptions().keys()))) + .thenAccept(topic -> asyncResponse.resume(new ArrayList<>(topic.getSubscriptions().keys()))) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. if (!isRedirectException(ex)) { @@ -1309,7 +1306,7 @@ protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean .thenAccept(partitionMetadata -> { if (partitionMetadata.partitions > 0) { final List> futures = - Lists.newArrayListWithCapacity(partitionMetadata.partitions); + new ArrayList<>(partitionMetadata.partitions); PartitionedManagedLedgerInfo partitionedManagedLedgerInfo = new PartitionedManagedLedgerInfo(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -1509,7 +1506,7 @@ protected void internalGetPartitionedStatsInternal(AsyncResponse asyncResponse, PartitionedTopicInternalStats stats = new PartitionedTopicInternalStats(partitionMetadata); - List> topicStatsFutureList = Lists.newArrayList(); + List> topicStatsFutureList = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { try { topicStatsFutureList.add(pulsar().getAdminClient().topics() @@ -1571,7 +1568,7 @@ protected void internalDeleteSubscription(AsyncResponse asyncResponse, String su getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenAcceptAsync(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -1792,7 +1789,7 @@ protected void internalDeleteSubscriptionForcefully(AsyncResponse asyncResponse, getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenAccept(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -1895,7 +1892,7 @@ protected void internalSkipAllMessages(AsyncResponse asyncResponse, String subNa return getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenCompose(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -2072,8 +2069,7 @@ protected void internalExpireMessagesForAllSubscriptions(AsyncResponse asyncResp partitionMetadata, expireTimeInSeconds, authoritative); } else { if (partitionMetadata.partitions > 0) { - final List> futures = - Lists.newArrayListWithCapacity(partitionMetadata.partitions); + final List> futures = new ArrayList<>(partitionMetadata.partitions); // expire messages for each partition topic for (int i = 0; i < partitionMetadata.partitions; i++) { @@ -2145,9 +2141,9 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy } PersistentTopic topic = (PersistentTopic) t; final List> futures = - Lists.newArrayListWithCapacity((int) topic.getReplicators().size()); + new ArrayList<>((int) topic.getReplicators().size()); List subNames = - Lists.newArrayListWithCapacity((int) topic.getReplicators().size() + new ArrayList<>((int) topic.getReplicators().size() + (int) topic.getSubscriptions().size()); subNames.addAll(topic.getReplicators().keys()); subNames.addAll(topic.getSubscriptions().keys()); @@ -2472,7 +2468,7 @@ protected void internalUpdateSubscriptionProperties(AsyncResponse asyncResponse, getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenAcceptAsync(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -2590,7 +2586,7 @@ protected void internalGetSubscriptionProperties(AsyncResponse asyncResponse, St getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenAcceptAsync(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List>> futures = Lists.newArrayList(); + final List>> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -3242,10 +3238,10 @@ protected CompletableFuture> in Map quotaMap = op .map(TopicPolicies::getBackLogQuotaMap) .map(map -> { - HashMap hashMap = Maps.newHashMap(); + HashMap hashMap = new HashMap<>(); map.forEach((key, value) -> hashMap.put(BacklogQuota.BacklogQuotaType.valueOf(key), value)); return hashMap; - }).orElse(Maps.newHashMap()); + }).orElse(new HashMap<>()); if (applied && quotaMap.isEmpty()) { quotaMap = getNamespacePolicies(namespaceName).backlog_quota_map; if (quotaMap.isEmpty()) { @@ -3804,7 +3800,7 @@ protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, bo if (partitionMetadata.partitions > 0) { Map messageIds = new ConcurrentHashMap<>(partitionMetadata.partitions); final List> futures = - Lists.newArrayListWithCapacity(partitionMetadata.partitions); + new ArrayList<>(partitionMetadata.partitions); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -3882,7 +3878,7 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St } else { if (partitionMetadata.partitions > 0) { return CompletableFuture.completedFuture(null).thenAccept(unused -> { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); // expire messages for each partition topic for (int i = 0; i < partitionMetadata.partitions; i++) { @@ -4148,7 +4144,7 @@ protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean au .thenAccept(partitionMetadata -> { final int numPartitions = partitionMetadata.partitions; if (numPartitions > 0) { - final List> futures = Lists.newArrayListWithCapacity(numPartitions); + final List> futures = new ArrayList<>(numPartitions); for (int i = 0; i < numPartitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -4958,7 +4954,7 @@ protected CompletableFuture>> internalGetSubscriptionType protected CompletableFuture internalSetSubscriptionTypesEnabled( Set subscriptionTypesEnabled, boolean isGlobal) { - List subTypes = Lists.newArrayList(); + List subTypes = new ArrayList<>(); subscriptionTypesEnabled.forEach(subscriptionType -> subTypes.add(SubType.valueOf(subscriptionType.name()))); return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenCompose(op -> { @@ -4975,7 +4971,7 @@ protected CompletableFuture internalRemoveSubscriptionTypesEnabled(boolean if (!op.isPresent()) { return CompletableFuture.completedFuture(null); } - op.get().setSubscriptionTypesEnabled(Lists.newArrayList()); + op.get().setSubscriptionTypesEnabled(new ArrayList<>()); op.get().setIsGlobal(isGlobal); return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, op.get()); }); @@ -5114,7 +5110,7 @@ protected void internalSetReplicatedSubscriptionStatus(AsyncResponse asyncRespon thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) .thenAccept(partitionMetadata -> { if (partitionMetadata.partitions > 0) { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); @@ -5252,7 +5248,7 @@ protected void internalGetReplicatedSubscriptionStatus(AsyncResponse asyncRespon .thenAccept(partitionMetadata -> { if (partitionMetadata.partitions > 0) { List> futures = new ArrayList<>(partitionMetadata.partitions); - Map status = Maps.newHashMap(); + Map status = new HashMap<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName partition = topicName.getPartition(i); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index a115b26407d18..0254ff395ba2a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -18,13 +18,12 @@ */ package org.apache.pulsar.broker.admin.impl; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.isNull; import static org.apache.commons.lang.StringUtils.defaultIfEmpty; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.Clock; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -123,12 +122,12 @@ public CompletableFuture postSchemaAsync(PostSchemaPayload payloa try { data = DefaultImplementation.getDefaultImplementation() .convertKeyValueDataStringToSchemaInfoSchema(payload.getSchema() - .getBytes(Charsets.UTF_8)); + .getBytes(StandardCharsets.UTF_8)); } catch (IOException conversionError) { throw new RestException(conversionError); } } else { - data = payload.getSchema().getBytes(Charsets.UTF_8); + data = payload.getSchema().getBytes(StandardCharsets.UTF_8); } return pulsar().getSchemaRegistryService() .putSchemaIfAbsent(getSchemaId(), @@ -148,7 +147,7 @@ public CompletableFuture> testCompati .thenCompose(strategy -> { String schemaId = getSchemaId(); return pulsar().getSchemaRegistryService().isCompatible(schemaId, - SchemaData.builder().data(payload.getSchema().getBytes(Charsets.UTF_8)) + SchemaData.builder().data(payload.getSchema().getBytes(StandardCharsets.UTF_8)) .isDeleted(false) .timestamp(clock.millis()).type(SchemaType.valueOf(payload.getType())) .user(defaultIfEmpty(clientAppId(), "")) @@ -164,7 +163,7 @@ public CompletableFuture getVersionBySchemaAsync(PostSchemaPayload payload String schemaId = getSchemaId(); return pulsar().getSchemaRegistryService() .findSchemaVersion(schemaId, - SchemaData.builder().data(payload.getSchema().getBytes(Charsets.UTF_8)) + SchemaData.builder().data(payload.getSchema().getBytes(StandardCharsets.UTF_8)) .isDeleted(false).timestamp(clock.millis()) .type(SchemaType.valueOf(payload.getType())) .user(defaultIfEmpty(clientAppId(), "")) @@ -185,7 +184,7 @@ private static GetSchemaResponse convertSchemaAndMetadataToGetSchemaResponse(Sch DefaultImplementation.getDefaultImplementation() .decodeKeyValueSchemaInfo(schemaAndMetadata.schema.toSchemaInfo())); } else { - schemaData = new String(schemaAndMetadata.schema.getData(), UTF_8); + schemaData = new String(schemaAndMetadata.schema.getData(), StandardCharsets.UTF_8); } return GetSchemaResponse.builder().version(getLongSchemaVersion(schemaAndMetadata.version)) .type(schemaAndMetadata.schema.getType()).timestamp(schemaAndMetadata.schema.getTimestamp()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index c2b8bb7220ba8..fdf0433b340cf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -20,7 +20,6 @@ package org.apache.pulsar.broker.admin.impl; import static org.apache.pulsar.common.naming.Constants.GLOBAL_CLUSTER; -import com.google.common.collect.Lists; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; @@ -249,7 +248,7 @@ protected CompletableFuture internalDeleteTenantAsyncForcefully(String ten } return tenantResources().getListOfNamespacesAsync(tenant) .thenApply(namespaces -> { - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); try { PulsarAdmin adminClient = pulsar().getAdminClient(); for (String namespace : namespaces) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index b2137d7433870..dbd70feb1c052 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -21,7 +21,6 @@ import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -92,7 +91,7 @@ protected void internalGetCoordinatorStats(AsyncResponse asyncResponse, boolean return; } List> transactionMetadataStoreInfoFutures = - Lists.newArrayList(); + new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { try { transactionMetadataStoreInfoFutures @@ -309,7 +308,7 @@ protected void internalGetSlowTransactions(AsyncResponse asyncResponse, return; } List>> completableFutures = - Lists.newArrayList(); + new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { try { completableFutures diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 9fd2020d48ab1..3fd09ddd594e3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -19,12 +19,12 @@ package org.apache.pulsar.broker.admin.v1; import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; -import com.google.common.collect.Lists; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -107,7 +107,7 @@ public void getTenantNamespaces(@Suspended AsyncResponse response, public List getNamespacesForCluster(@PathParam("property") String tenant, @PathParam("cluster") String cluster) { validateTenantOperation(tenant, TenantOperation.LIST_NAMESPACES); - List namespaces = Lists.newArrayList(); + List namespaces = new ArrayList<>(); if (!clusters().contains(cluster)) { log.warn("[{}] Failed to get namespace list for tenant: {}/{} - Cluster does not exist", clientAppId(), tenant, cluster); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java index 34e9eadd71665..fef5eda009bde 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java @@ -19,11 +19,11 @@ package org.apache.pulsar.broker.admin.v1; -import com.google.common.collect.Lists; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -200,7 +200,7 @@ public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("pr return; } - final List>> futures = Lists.newArrayList(); + final List>> futures = new ArrayList<>(); final List boundaries = policies.bundles.getBoundaries(); for (int i = 0; i < boundaries.size() - 1; i++) { final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); @@ -222,7 +222,7 @@ public void getList(@Suspended final AsyncResponse asyncResponse, @PathParam("pr if (ex != null) { resumeAsyncResponseExceptionally(asyncResponse, ex); } else { - final List topics = Lists.newArrayList(); + final List topics = new ArrayList<>(); for (int i = 0; i < futures.size(); i++) { List topicList = futures.get(i).join(); if (topicList != null) { @@ -267,7 +267,7 @@ public List getListFromBundle(@PathParam("property") String property, @P } NamespaceBundle nsBundle = validateNamespaceBundleOwnership(fqnn, policies.bundles, bundleRange, true, true); - final List topicList = Lists.newArrayList(); + final List topicList = new ArrayList<>(); pulsar().getBrokerService().forEachTopic(topic -> { TopicName topicName = TopicName.get(topic.getName()); if (nsBundle.includes(topicName)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 0ebfabb78913b..dace261ceb6ae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.admin.v2; import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; -import com.google.common.collect.Sets; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -2319,7 +2318,7 @@ public void setSubscriptionTypesEnabled( public void removeSubscriptionTypesEnabled(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); - internalSetSubscriptionTypesEnabled(Sets.newHashSet()); + internalSetSubscriptionTypesEnabled(new HashSet<>()); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index 82da5f91263a4..5b0deccd0378b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -19,12 +19,12 @@ package org.apache.pulsar.broker.admin.v2; -import com.google.common.collect.Lists; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -233,7 +233,7 @@ public void getPartitionedStats( } NonPersistentPartitionedTopicStatsImpl stats = new NonPersistentPartitionedTopicStatsImpl(partitionMetadata); - List> topicStatsFutureList = Lists.newArrayList(); + List> topicStatsFutureList = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { try { topicStatsFutureList @@ -369,7 +369,7 @@ public void getList( return; } - final List>> futures = Lists.newArrayList(); + final List>> futures = new ArrayList<>(); final List boundaries = policies.bundles.getBoundaries(); for (int i = 0; i < boundaries.size() - 1; i++) { final String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); @@ -390,7 +390,7 @@ public void getList( if (ex != null) { resumeAsyncResponseExceptionally(asyncResponse, ex); } else { - final List topics = Lists.newArrayList(); + final List topics = new ArrayList<>(); for (int i = 0; i < futures.size(); i++) { List topicList = futures.get(i).join(); if (topicList != null) { @@ -458,7 +458,7 @@ public void getListFromBundle( asyncResponse.resume(Collections.emptyList()); return; } - final List topicList = Lists.newArrayList(); + final List topicList = new ArrayList<>(); String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange(); ConcurrentOpenHashMap topicMap = bundleTopics.get(bundleKey); if (topicMap != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 83b113df36b6e..11d663322be52 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -59,7 +59,7 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T // always going to be in FIFO order, then we can avoid pulling all the messages in // tracker. Instead, we use the lookahead for detection and pause the read from // the cursor if the delays are fixed. - public static final long DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES = 50_000; + private final long fixedDelayDetectionLookahead; // This is the timestamp of the message with the highest delivery time // If new added messages are lower than this, it means the delivery is requested @@ -70,17 +70,22 @@ public class InMemoryDelayedDeliveryTracker implements DelayedDeliveryTracker, T private boolean messagesHaveFixedDelay = true; InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, - boolean isDelayedDeliveryDeliverAtTimeStrict) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); + boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead) { + this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, + fixedDelayDetectionLookahead); } InMemoryDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, - long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict) { + long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead) { this.dispatcher = dispatcher; this.timer = timer; this.tickTimeMillis = tickTimeMillis; this.clock = clock; this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; + this.fixedDelayDetectionLookahead = fixedDelayDetectionLookahead; } /** @@ -283,8 +288,9 @@ public void close() { @Override public boolean shouldPauseAllDeliveries() { // Pause deliveries if we know all delays are fixed within the lookahead window - return messagesHaveFixedDelay - && priorityQueue.size() >= DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES + return fixedDelayDetectionLookahead > 0 + && messagesHaveFixedDelay + && priorityQueue.size() >= fixedDelayDetectionLookahead && !hasMessageAvailable(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java index 5c04a6d53b257..7bf0ca87c40c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java @@ -33,18 +33,21 @@ public class InMemoryDelayedDeliveryTrackerFactory implements DelayedDeliveryTra private boolean isDelayedDeliveryDeliverAtTimeStrict; + private long fixedDelayDetectionLookahead; + @Override public void initialize(ServiceConfiguration config) { this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-delayed-delivery"), config.getDelayedDeliveryTickTimeMillis(), TimeUnit.MILLISECONDS); this.tickTimeMillis = config.getDelayedDeliveryTickTimeMillis(); this.isDelayedDeliveryDeliverAtTimeStrict = config.isDelayedDeliveryDeliverAtTimeStrict(); + this.fixedDelayDetectionLookahead = config.getDelayedDeliveryFixedDelayDetectionLookahead(); } @Override public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { return new InMemoryDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, - isDelayedDeliveryDeliverAtTimeStrict); + isDelayedDeliveryDeliverAtTimeStrict, fixedDelayDetectionLookahead); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 1e48052a6f736..26528491f1ebb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -177,7 +177,7 @@ public static double getTotalNicLimit(List nics, BitRateUnit bitRateUnit log.error("[LinuxInfo] Failed to get total nic limit.", e); return 0d; } - }).sum(), BitRateUnit.Bit); + }).sum(), BitRateUnit.Megabit); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index aa801f2187d4d..a298558f34502 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -20,10 +20,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; -import com.beust.jcommander.internal.Lists; import io.netty.util.concurrent.FastThreadLocal; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -433,7 +433,7 @@ public static CompletableFuture> getAntiAffinityNamespaceOw } final String antiAffinityGroup = policies.get().namespaceAntiAffinityGroup; final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>(); - final List> futures = Lists.newArrayList(); + final List> futures = new ArrayList<>(); brokerToNamespaceToBundleRange.forEach((broker, nsToBundleRange) -> { nsToBundleRange.forEach((ns, bundleRange) -> { if (bundleRange.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceEphemeralData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceEphemeralData.java index 2f1262f5bf225..e86007b64460d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceEphemeralData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceEphemeralData.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.namespace; -import com.google.common.collect.Maps; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.validation.constraints.NotNull; import lombok.Data; @@ -49,16 +49,16 @@ public NamespaceEphemeralData(String brokerUrl, String brokerUrlTls, String http this.httpUrlTls = httpUrlTls; this.disabled = disabled; if (advertisedListeners == null) { - this.advertisedListeners = Collections.EMPTY_MAP; + this.advertisedListeners = Collections.emptyMap(); } else { - this.advertisedListeners = Maps.newHashMap(advertisedListeners); + this.advertisedListeners = new HashMap<>(advertisedListeners); } } @NotNull public Map getAdvertisedListeners() { if (this.advertisedListeners == null) { - return Collections.EMPTY_MAP; + return Collections.emptyMap(); } return Collections.unmodifiableMap(this.advertisedListeners); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 33d7057c4bd36..5c1cb283d6990 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -19,21 +19,21 @@ package org.apache.pulsar.broker.namespace; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; -import com.google.common.collect.Lists; import com.google.common.hash.Hashing; import io.prometheus.client.Counter; import java.net.URI; import java.net.URL; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -549,7 +549,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } try { - checkNotNull(candidateBroker); + Objects.requireNonNull(candidateBroker); if (candidateBroker.equals(pulsar.getSafeWebServiceAddress())) { // Load manager decided that the local broker should try to become the owner @@ -865,8 +865,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, return; } - checkNotNull(splittedBundles.getLeft()); - checkNotNull(splittedBundles.getRight()); + Objects.requireNonNull(splittedBundles.getLeft()); + Objects.requireNonNull(splittedBundles.getRight()); checkArgument(splittedBundles.getRight().size() == splitBoundaries.size() + 1, "bundle has to be split in " + (splitBoundaries.size() + 1) + " bundles"); NamespaceName nsname = bundle.getNamespaceObject(); @@ -878,7 +878,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, try { // take ownership of newly split bundles for (NamespaceBundle sBundle : splittedBundles.getRight()) { - checkNotNull(ownershipCache.tryAcquiringOwnership(sBundle)); + Objects.requireNonNull(ownershipCache.tryAcquiringOwnership(sBundle)); } updateNamespaceBundles(nsname, splittedBundles.getLeft()) .thenRun(() -> { @@ -968,8 +968,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, * @throws Exception */ private CompletableFuture updateNamespaceBundles(NamespaceName nsname, NamespaceBundles nsBundles) { - checkNotNull(nsname); - checkNotNull(nsBundles); + Objects.requireNonNull(nsname); + Objects.requireNonNull(nsBundles); LocalPolicies localPolicies = nsBundles.toLocalPolicies(); @@ -1088,7 +1088,7 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) { } public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener... listeners) { - checkNotNull(listeners); + Objects.requireNonNull(listeners); for (NamespaceBundleOwnershipListener listener : listeners) { if (listener != null) { bundleOwnershipListeners.add(listener); @@ -1208,9 +1208,9 @@ public CompletableFuture> getPartitions(NamespaceName namespaceName .listPartitionedTopicsAsync(namespaceName, topicDomain) .thenCompose(topics -> { CompletableFuture> result = new CompletableFuture<>(); - List resultPartitions = Collections.synchronizedList(Lists.newArrayList()); + List resultPartitions = Collections.synchronizedList(new ArrayList<>()); if (CollectionUtils.isNotEmpty(topics)) { - List>> futures = Lists.newArrayList(); + List>> futures = new ArrayList<>(); for (String topic : topics) { CompletableFuture> future = getPartitionsForTopic(TopicName.get(topic)); futures.add(future); @@ -1232,7 +1232,7 @@ public CompletableFuture> getPartitions(NamespaceName namespaceName private CompletableFuture> getPartitionsForTopic(TopicName topicName) { return pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName).thenCompose(meta -> { - List result = Lists.newArrayList(); + List result = new ArrayList<>(); for (int i = 0; i < meta.partitions; i++) { result.add(topicName.getPartition(i).toString()); } @@ -1255,7 +1255,7 @@ public CompletableFuture> getListOfNonPersistentTopics(NamespaceNam } else { // Non-persistent topics don't have managed ledgers so we have to retrieve them from local // cache. - List topics = Lists.newArrayList(); + List topics = new ArrayList<>(); synchronized (pulsar.getBrokerService().getMultiLayerTopicMap()) { if (pulsar.getBrokerService().getMultiLayerTopicMap() .containsKey(namespaceName.toString())) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index 4f60512a7966a..e0cdd129a8f03 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -21,8 +21,8 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -223,7 +223,7 @@ public CompletableFuture removeOwnership(NamespaceBundle bundle) { * NamespaceBundles to remove from ownership cache */ public CompletableFuture removeOwnership(NamespaceBundles bundles) { - List> allFutures = Lists.newArrayList(); + List> allFutures = new ArrayList<>(); for (NamespaceBundle bundle : bundles.getBundles()) { if (getOwnedBundle(bundle) == null) { // continue diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index 4bb1bc8ab243a..c74681fdb731a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -53,7 +53,7 @@ * * @see PulsarService */ -public class ResourceGroupService { +public class ResourceGroupService implements AutoCloseable{ /** * Default constructor. */ @@ -303,6 +303,21 @@ public ResourceGroup getNamespaceResourceGroup(NamespaceName namespaceName) { return this.namespaceToRGsMap.get(namespaceName); } + @Override + public void close() throws Exception { + if (aggregateLocalUsagePeriodicTask != null) { + aggregateLocalUsagePeriodicTask.cancel(true); + } + if (calculateQuotaPeriodicTask != null) { + calculateQuotaPeriodicTask.cancel(true); + } + resourceGroupsMap.clear(); + tenantToRGsMap.clear(); + namespaceToRGsMap.clear(); + topicProduceStats.clear(); + topicConsumeStats.clear(); + } + /** * Increments usage stats for the resource groups associated with the given namespace and tenant. * Expected to be called when a message is produced or consumed on a topic, or when we calculate @@ -565,17 +580,17 @@ protected void aggregateResourceGroupLocalUsages() { ServiceConfiguration config = pulsar.getConfiguration(); long newPeriodInSeconds = config.getResourceUsageTransportPublishIntervalInSecs(); if (newPeriodInSeconds != this.aggregateLocalUsagePeriodInSeconds) { - if (this.aggreagteLocalUsagePeriodicTask == null) { + if (this.aggregateLocalUsagePeriodicTask == null) { log.error("aggregateResourceGroupLocalUsages: Unable to find running task to cancel when " + "publish period changed from {} to {} {}", this.aggregateLocalUsagePeriodInSeconds, newPeriodInSeconds, timeUnitScale); } else { - boolean cancelStatus = this.aggreagteLocalUsagePeriodicTask.cancel(true); + boolean cancelStatus = this.aggregateLocalUsagePeriodicTask.cancel(true); log.info("aggregateResourceGroupLocalUsages: Got status={} in cancel of periodic " + "when publish period changed from {} to {} {}", cancelStatus, this.aggregateLocalUsagePeriodInSeconds, newPeriodInSeconds, timeUnitScale); } - this.aggreagteLocalUsagePeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( + this.aggregateLocalUsagePeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( catchingAndLoggingThrowables(this::aggregateResourceGroupLocalUsages), newPeriodInSeconds, newPeriodInSeconds, @@ -680,7 +695,7 @@ private void initialize() { ServiceConfiguration config = this.pulsar.getConfiguration(); long periodInSecs = config.getResourceUsageTransportPublishIntervalInSecs(); this.aggregateLocalUsagePeriodInSeconds = this.resourceUsagePublishPeriodInSeconds = periodInSecs; - this.aggreagteLocalUsagePeriodicTask = this.pulsar.getExecutor().scheduleAtFixedRate( + this.aggregateLocalUsagePeriodicTask = this.pulsar.getExecutor().scheduleAtFixedRate( catchingAndLoggingThrowables(this::aggregateResourceGroupLocalUsages), periodInSecs, periodInSecs, @@ -737,7 +752,7 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie // The task that periodically re-calculates the quota budget for local usage. - private ScheduledFuture aggreagteLocalUsagePeriodicTask; + private ScheduledFuture aggregateLocalUsagePeriodicTask; private long aggregateLocalUsagePeriodInSeconds; // The task that periodically re-calculates the quota budget for local usage. @@ -840,4 +855,14 @@ ConcurrentHashMap getTopicConsumeStats() { ConcurrentHashMap getTopicProduceStats() { return this.topicProduceStats; } + + @VisibleForTesting + ScheduledFuture getAggregateLocalUsagePeriodicTask() { + return this.aggregateLocalUsagePeriodicTask; + } + + @VisibleForTesting + ScheduledFuture getCalculateQuotaPeriodicTask() { + return this.calculateQuotaPeriodicTask; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 29710067a61d4..df02bbd85d470 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -107,6 +107,9 @@ public int filterEntriesForConsumer(Optional optMetadataArray long totalBytes = 0; int totalChunkedMessages = 0; int totalEntries = 0; + int filteredMessageCount = 0; + int filteredEntryCount = 0; + long filteredBytesCount = 0; final boolean hasFilter = CollectionUtils.isNotEmpty(entryFilters); List entriesToFiltered = hasFilter ? new ArrayList<>() : null; List entriesToRedeliver = hasFilter ? new ArrayList<>() : null; @@ -135,6 +138,9 @@ public int filterEntriesForConsumer(Optional optMetadataArray // FilterResult will be always `ACCEPTED` when there is No Filter // dont need to judge whether `hasFilter` is true or not. this.filterRejectedMsgs.add(entryMsgCnt); + filteredEntryCount++; + filteredMessageCount += entryMsgCnt; + filteredBytesCount += metadataAndPayload.readableBytes(); entry.release(); continue; } else if (filterResult == EntryFilter.FilterResult.RESCHEDULE) { @@ -143,6 +149,9 @@ public int filterEntriesForConsumer(Optional optMetadataArray // FilterResult will be always `ACCEPTED` when there is No Filter // dont need to judge whether `hasFilter` is true or not. this.filterRescheduledMsgs.add(entryMsgCnt); + filteredEntryCount++; + filteredMessageCount += entryMsgCnt; + filteredBytesCount += metadataAndPayload.readableBytes(); entry.release(); continue; } @@ -231,6 +240,11 @@ public int filterEntriesForConsumer(Optional optMetadataArray } + if (serviceConfig.isDispatchThrottlingForFilteredEntriesEnabled()) { + acquirePermitsForDeliveredMessages(subscription.getTopic(), cursor, filteredEntryCount, + filteredMessageCount, filteredBytesCount); + } + sendMessageInfo.setTotalMessages(totalMessages); sendMessageInfo.setTotalBytes(totalBytes); sendMessageInfo.setTotalChunkedMessages(totalChunkedMessages); @@ -243,6 +257,19 @@ private void individualAcknowledgeMessageIfNeeded(Position position, Map + rateLimiter.tryDispatchPermit(permits, totalBytesSent)); + topic.getDispatchRateLimiter().ifPresent(rateLimter -> + rateLimter.tryDispatchPermit(permits, totalBytesSent)); + getRateLimiter().ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(permits, totalBytesSent)); + } + } + /** * Determine whether the number of consumers on the subscription reaches the threshold. * @return diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index a3df1498698f3..428c54ecc191d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -147,7 +145,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener entryFilters; + protected Map entryFilters; public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; @@ -187,7 +185,7 @@ public EntryFilters getEntryFiltersPolicy() { return this.topicPolicies.getEntryFilters().get(); } - public ImmutableMap getEntryFilters() { + public Map getEntryFilters() { return this.entryFilters; } @@ -251,7 +249,7 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { topicPolicies.getRetentionPolicies().updateNamespaceValue(namespacePolicies.retention_policies); topicPolicies.getCompactionThreshold().updateNamespaceValue(namespacePolicies.compaction_threshold); topicPolicies.getReplicationClusters().updateNamespaceValue( - Lists.newArrayList(CollectionUtils.emptyIfNull(namespacePolicies.replication_clusters))); + new ArrayList<>(CollectionUtils.emptyIfNull(namespacePolicies.replication_clusters))); topicPolicies.getMaxUnackedMessagesOnConsumer() .updateNamespaceValue(namespacePolicies.max_unacked_messages_per_consumer); topicPolicies.getMaxUnackedMessagesOnSubscription() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 805d00adca6bb..b11538a9217e2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -245,7 +245,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo * The topic on which all producers should be disconnected */ private void disconnectProducers(PersistentTopic persistentTopic) { - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); Map producers = persistentTopic.getProducers(); producers.values().forEach(producer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a3cef4bd0276b..2f6cb020b7085 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -19,15 +19,11 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Queues; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; @@ -52,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CancellationException; @@ -277,7 +274,7 @@ public class BrokerService implements Closeable { private boolean preciseTopicPublishRateLimitingEnable; private final LongAdder pausedConnections = new LongAdder(); private BrokerInterceptor interceptor; - private ImmutableMap entryFilters; + private Map entryFilters; private TopicFactory topicFactory; private Set brokerEntryMetadataInterceptors; @@ -1408,6 +1405,22 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S return topicFuture; } + CompletableFuture> fetchTopicPropertiesAsync(TopicName topicName) { + if (!topicName.isPartitioned()) { + return managedLedgerFactory.getManagedLedgerPropertiesAsync(topicName.getPersistenceNamingEncoding()); + } else { + TopicName partitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + return fetchPartitionedTopicMetadataAsync(partitionedTopicName) + .thenCompose(metadata -> { + if (metadata.partitions == PartitionedTopicMetadata.NON_PARTITIONED) { + return managedLedgerFactory.getManagedLedgerPropertiesAsync( + topicName.getPersistenceNamingEncoding()); + } + return CompletableFuture.completedFuture(metadata.properties); + }); + } + } + private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean createIfMissing, CompletableFuture> topicFuture, Map properties) { @@ -1415,7 +1428,21 @@ private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean pulsar.getNamespaceService().isServiceUnitActiveAsync(topicName) .thenAccept(isActive -> { if (isActive) { - createPersistentTopic(topic, createIfMissing, topicFuture, properties); + CompletableFuture> propertiesFuture; + if (properties == null) { + //Read properties from storage when loading topic. + propertiesFuture = fetchTopicPropertiesAsync(topicName); + } else { + propertiesFuture = CompletableFuture.completedFuture(properties); + } + propertiesFuture.thenAccept(finalProperties -> + createPersistentTopic(topic, createIfMissing, topicFuture, finalProperties) + ).exceptionally(throwable -> { + log.warn("[{}] Read topic property failed", topic, throwable); + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(throwable); + return null; + }); } else { // namespace is being unloaded String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic); @@ -1604,20 +1631,20 @@ public CompletableFuture getManagedLedgerConfig(TopicName t managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( IsolatedBookieEnsemblePlacementPolicy.class); if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); } else if (isSystemTopic(topicName)) { - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); properties.put(IsolatedBookieEnsemblePlacementPolicy .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); } else { - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); @@ -1626,7 +1653,7 @@ public CompletableFuture getManagedLedgerConfig(TopicName t if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( IsolatedBookieEnsemblePlacementPolicy.class); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, @@ -1744,7 +1771,7 @@ private void addTopicToStatsMaps(TopicName topicName, Topic topic) { } public void refreshTopicToStatsMaps(NamespaceBundle oldBundle) { - checkNotNull(oldBundle); + Objects.requireNonNull(oldBundle); try { // retrieve all topics under existing old bundle List topics = getAllTopicsFromNamespaceBundle(oldBundle.getNamespaceObject().toString(), @@ -1974,7 +2001,7 @@ public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, */ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, boolean closeWithoutWaitingClientDisconnect) { - List> closeFutures = Lists.newArrayList(); + List> closeFutures = new ArrayList<>(); topics.forEach((name, topicFuture) -> { TopicName topicName = TopicName.get(name); if (serviceUnit.includes(topicName)) { @@ -2677,7 +2704,7 @@ private void createDynamicConfigPathIfNotExist() { // create dynamic-config if not exist. if (!configCache.isPresent()) { pulsar().getPulsarResources().getDynamicConfigResources() - .setDynamicConfigurationWithCreate(n -> Maps.newHashMap()); + .setDynamicConfigurationWithCreate(n -> new HashMap<>()); } } catch (Exception e) { log.warn("Failed to read dynamic broker configuration", e); @@ -2698,7 +2725,7 @@ private void updateDynamicServiceConfiguration() { // create dynamic-config if not exist. if (!configCache.isPresent()) { pulsar().getPulsarResources().getDynamicConfigResources() - .setDynamicConfigurationWithCreate(n -> Maps.newHashMap()); + .setDynamicConfigurationWithCreate(n -> new HashMap<>()); } } catch (Exception e) { log.warn("Failed to read dynamic broker configuration", e); @@ -2736,7 +2763,7 @@ public static List getDynamicConfiguration() { } public Map getRuntimeConfiguration() { - Map configMap = Maps.newHashMap(); + Map configMap = new HashMap<>(); ConcurrentOpenHashMap runtimeConfigurationMap = getRuntimeConfigurationMap(); runtimeConfigurationMap.forEach((key, value) -> { configMap.put(key, String.valueOf(value)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 7eae83ba25031..767c7bb92747d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -22,7 +22,6 @@ import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.AtomicDouble; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.Promise; @@ -87,9 +86,9 @@ public class Consumer { private final LongAdder bytesOutCounter; private final Rate messageAckRate; - private long lastConsumedTimestamp; - private long lastAckedTimestamp; - private long lastConsumedFlowTimestamp; + private volatile long lastConsumedTimestamp; + private volatile long lastAckedTimestamp; + private volatile long lastConsumedFlowTimestamp; private Rate chunkedMessageRate; // Represents how many messages we can safely send to the consumer without @@ -994,7 +993,7 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { public void redeliverUnacknowledgedMessages(List messageIds) { int totalRedeliveryMessages = 0; - List pendingPositions = Lists.newArrayList(); + List pendingPositions = new ArrayList<>(); for (MessageIdData msg : messageIds) { PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId()); LongPair longPair = pendingAcks.get(position.getLedgerId(), position.getEntryId()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java index 7a4700a90a6d6..8704e21ae7556 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java @@ -18,8 +18,9 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.apache.bookkeeper.mledger.Entry; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -34,7 +35,7 @@ public class EntryFilterSupport { * Entry filters in Broker. * Not set to final, for the convenience of testing mock. */ - protected ImmutableList entryFilters; + protected List entryFilters; protected final FilterContext filterContext; protected final Subscription subscription; @@ -45,19 +46,20 @@ public EntryFilterSupport(Subscription subscription) { .getBrokerService().getEntryFilters()) && !subscription.getTopic().getBrokerService().pulsar() .getConfiguration().isAllowOverrideEntryFilters()) { - this.entryFilters = subscription.getTopic().getBrokerService().getEntryFilters().values().asList(); + this.entryFilters = subscription.getTopic().getBrokerService().getEntryFilters().values().stream() + .toList(); } else { - ImmutableMap entryFiltersMap = + Map entryFiltersMap = subscription.getTopic().getEntryFilters(); if (entryFiltersMap != null) { - this.entryFilters = subscription.getTopic().getEntryFilters().values().asList(); + this.entryFilters = subscription.getTopic().getEntryFilters().values().stream().toList(); } else { - this.entryFilters = ImmutableList.of(); + this.entryFilters = Collections.emptyList(); } } this.filterContext = new FilterContext(); } else { - this.entryFilters = ImmutableList.of(); + this.entryFilters = Collections.emptyList(); this.filterContext = FilterContext.FILTER_CONTEXT_DISABLED; } } @@ -82,7 +84,7 @@ private void fillContext(FilterContext context, MessageMetadata msgMetadata, private static EntryFilter.FilterResult getFilterResult(FilterContext filterContext, Entry entry, - ImmutableList entryFilters) { + List entryFilters) { for (EntryFilter entryFilter : entryFilters) { EntryFilter.FilterResult filterResult = entryFilter.filterEntry(entry, filterContext); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index b5d87a46cfc09..62182f6e84f49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service; -import static com.google.common.base.Preconditions.checkNotNull; import static com.scurrilous.circe.checksum.Crc32cIntChecksum.computeChecksum; import static org.apache.pulsar.broker.service.AbstractReplicator.REPL_PRODUCER_NAME_DELIMITER; import static org.apache.pulsar.common.protocol.Commands.hasChecksum; @@ -106,7 +105,7 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.topic = topic; this.cnx = cnx; this.producerId = producerId; - this.producerName = checkNotNull(producerName); + this.producerName = Objects.requireNonNull(producerName); this.userProvidedProducerName = userProvidedProducerName; this.epoch = epoch; this.closeFuture = new CompletableFuture<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index ff74cf839aef8..0babf2fe01223 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -18,14 +18,14 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.util.ReferenceCountUtil; import java.io.Closeable; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import org.apache.pulsar.broker.PulsarService; @@ -65,12 +65,12 @@ public PulsarStats(PulsarService pulsar) { this.nsStats = new NamespaceStats(pulsar.getConfig().getStatsUpdateFrequencyInSecs()); this.clusterReplicationMetrics = new ClusterReplicationMetrics(pulsar.getConfiguration().getClusterName(), pulsar.getConfiguration().isReplicationMetricsEnabled()); - this.bundleStats = Maps.newConcurrentMap(); - this.tempMetricsCollection = Lists.newArrayList(); - this.metricsCollection = Lists.newArrayList(); + this.bundleStats = new ConcurrentHashMap<>(); + this.tempMetricsCollection = new ArrayList<>(); + this.metricsCollection = new ArrayList<>(); this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar.getConfiguration().getClusterName(), pulsar.getAdvertisedAddress()); - this.tempNonPersistentTopics = Lists.newArrayList(); + this.tempNonPersistentTopics = new ArrayList<>(); this.exposePublisherStats = pulsar.getConfiguration().isExposePublisherStats(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index b3a5e8c2b8b96..9c3c848897128 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.service; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.HashSet; import java.util.List; @@ -27,6 +26,7 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; @@ -559,7 +559,7 @@ public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { public void registerListener(TopicName topicName, TopicPolicyListener listener) { listeners.compute(topicName, (k, topicListeners) -> { if (topicListeners == null) { - topicListeners = Lists.newCopyOnWriteArrayList(); + topicListeners = new CopyOnWriteArrayList<>(); } topicListeners.add(listener); return topicListeners; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 72e5d6a9228f0..c0f931bd6a553 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.ImmutableMap; import io.netty.buffer.ByteBuf; import java.util.Map; import java.util.Optional; @@ -250,7 +249,7 @@ CompletableFuture createSubscription(String subscriptionName, Init EntryFilters getEntryFiltersPolicy(); - ImmutableMap getEntryFilters(); + Map getEntryFilters(); BacklogQuota getBacklogQuota(BacklogQuotaType backlogQuotaType); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index a3da80080ffc0..05e42c1b64d6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -23,8 +23,6 @@ import static org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; @@ -34,6 +32,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -394,7 +393,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c CompletableFuture closeClientFuture = new CompletableFuture<>(); if (closeIfClientsConnected) { - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); producers.values().forEach(producer -> futures.add(producer.disconnect())); subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); @@ -415,7 +414,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c if (currentUsageCount() == 0) { isFenced = true; - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); if (failIfHasSubscriptions) { if (!subscriptions.isEmpty()) { @@ -485,7 +484,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect lock.writeLock().unlock(); } - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); producers.values().forEach(producer -> futures.add(producer.disconnect())); @@ -531,7 +530,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect } public CompletableFuture stopReplProducers() { - List> closeFutures = Lists.newArrayList(); + List> closeFutures = new ArrayList<>(); replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); return FutureUtil.waitForAll(closeFutures); } @@ -551,7 +550,7 @@ public CompletableFuture checkReplication() { String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); // Check for missing replicators for (String cluster : configuredClusters) { @@ -911,7 +910,7 @@ public CompletableFuture getInternalStats(boolean PersistentTopicInternalStats stats = new PersistentTopicInternalStats(); stats.entriesAddedCounter = ENTRIES_ADDED_COUNTER_UPDATER.get(this); - stats.cursors = Maps.newTreeMap(); + stats.cursors = new TreeMap<>(); subscriptions.forEach((name, subs) -> stats.cursors.put(name, new CursorStats())); replicators.forEach((name, subs) -> stats.cursors.put(name, new CursorStats())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 02d2e725379b6..15b42fedd38ab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -684,7 +684,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis totalBytesSent += sendMessageInfo.getTotalBytes(); } - acquirePermitsForDeliveredMessages(totalEntries, totalMessagesSent, totalBytesSent); + acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); if (entriesToDispatch > 0) { if (log.isDebugEnabled()) { @@ -700,23 +700,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis return true; } - private void acquirePermitsForDeliveredMessages(long totalEntries, long totalMessagesSent, long totalBytesSent) { - // acquire message-dispatch permits for already delivered messages - long permits = dispatchThrottlingOnBatchMessageEnabled ? totalEntries : totalMessagesSent; - if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) { - if (topic.getBrokerDispatchRateLimiter().isPresent()) { - topic.getBrokerDispatchRateLimiter().get().tryDispatchPermit(permits, totalBytesSent); - } - if (topic.getDispatchRateLimiter().isPresent()) { - topic.getDispatchRateLimiter().get().tryDispatchPermit(permits, totalBytesSent); - } - - if (dispatchRateLimiter.isPresent()) { - dispatchRateLimiter.get().tryDispatchPermit(permits, totalBytesSent); - } - } - } - private boolean sendChunkedMessagesToConsumers(ReadType readType, List entries, MessageMetadata[] metadataArray) { @@ -775,7 +758,7 @@ private boolean sendChunkedMessagesToConsumers(ReadType readType, totalBytesSent += sendMessageInfo.getTotalBytes(); } - acquirePermitsForDeliveredMessages(totalEntries, totalMessagesSent, totalBytesSent); + acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); return numConsumers.get() == 0; // trigger a new readMoreEntries() call } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 47830e669af4e..3ba7a82aa5e35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -18,12 +18,12 @@ */ package org.apache.pulsar.broker.service.persistent; -import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import io.netty.util.Recycler; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledFuture; @@ -221,23 +221,8 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e redeliveryTracker, epoch) .addListener(future -> { if (future.isSuccess()) { - int permits = dispatchThrottlingOnBatchMessageEnabled ? entries.size() - : sendMessageInfo.getTotalMessages(); - // acquire message-dispatch permits for already delivered messages - if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) { - if (topic.getBrokerDispatchRateLimiter().isPresent()) { - topic.getBrokerDispatchRateLimiter().get().tryDispatchPermit(permits, - sendMessageInfo.getTotalBytes()); - } - - if (topic.getDispatchRateLimiter().isPresent()) { - topic.getDispatchRateLimiter().get().tryDispatchPermit(permits, - sendMessageInfo.getTotalBytes()); - } - dispatchRateLimiter.ifPresent(rateLimiter -> - rateLimiter.tryDispatchPermit(permits, - sendMessageInfo.getTotalBytes())); - } + acquirePermitsForDeliveredMessages(topic, cursor, entries.size(), + sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes()); // Schedule a new read batch operation only after the previous batch has been written to the socket. topic.getBrokerService().getTopicOrderedExecutor().executeOrdered(topicName, @@ -520,7 +505,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep } } - checkNotNull(c); + Objects.requireNonNull(c); // Reduce read batch size to avoid flooding bookies with retries readBatchSize = serviceConfig.getDispatcherMinReadBatchSize(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 8aabc7bf3d8af..9b1cd75282a4d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -26,8 +26,10 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerException.LedgerNotExistException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; @@ -191,7 +193,28 @@ public void findEntryFailed(ManagedLedgerException exception, Optional && (exception instanceof NonRecoverableLedgerException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", topicName, subName, failedReadPosition, exception.getMessage()); - findEntryComplete(failedReadPosition.get(), ctx); + if (exception instanceof LedgerNotExistException) { + long failedLedgerId = failedReadPosition.get().getLedgerId(); + ManagedLedgerImpl ledger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + Position lastPositionInLedger = ledger.getOptionalLedgerInfo(failedLedgerId) + .map(ledgerInfo -> PositionImpl.get(failedLedgerId, ledgerInfo.getEntries() - 1)) + .orElseGet(() -> { + Long nextExistingLedger = ledger.getNextValidLedger(failedReadPosition.get().getLedgerId()); + if (nextExistingLedger == null) { + log.info("[{}] [{}] Couldn't find next next valid ledger for expiry monitor when find " + + "entry failed {}", ledger.getName(), ledger.getName(), + failedReadPosition); + return (PositionImpl) failedReadPosition.get(); + } else { + return PositionImpl.get(nextExistingLedger, -1); + } + }); + log.info("[{}][{}] ledger not existed, will complete the last position of the non-existed" + + " ledger:{}", topicName, subName, lastPositionInLedger); + findEntryComplete(lastPositionInLedger, ctx); + } else { + findEntryComplete(failedReadPosition.get(), ctx); + } } expirationCheckInProgress = FALSE; updateRates(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 024ed8581ef1c..5eb553106e679 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -296,19 +296,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis } // acquire message-dispatch permits for already delivered messages - if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) { - long permits = dispatchThrottlingOnBatchMessageEnabled ? totalEntries : totalMessagesSent; - if (topic.getBrokerDispatchRateLimiter().isPresent()) { - topic.getBrokerDispatchRateLimiter().get().tryDispatchPermit(permits, totalBytesSent); - } - if (topic.getDispatchRateLimiter().isPresent()) { - topic.getDispatchRateLimiter().get().tryDispatchPermit(permits, totalBytesSent); - } - - if (dispatchRateLimiter.isPresent()) { - dispatchRateLimiter.get().tryDispatchPermit(permits, totalBytesSent); - } - } + acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); stuckConsumers.clear(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index e5d6251d177de..855bec48527e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -21,7 +21,6 @@ import static org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.base.Predicate; import io.netty.buffer.ByteBuf; import java.util.Collections; import java.util.LinkedHashMap; @@ -34,6 +33,7 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; @@ -107,9 +107,9 @@ public class PersistentSubscription extends AbstractSubscription implements Subs private volatile int isFenced = FALSE; private PersistentMessageExpiryMonitor expiryMonitor; - private long lastExpireTimestamp = 0L; - private long lastConsumedFlowTimestamp = 0L; - private long lastMarkDeleteAdvancedTimestamp = 0L; + private volatile long lastExpireTimestamp = 0L; + private volatile long lastConsumedFlowTimestamp = 0L; + private volatile long lastMarkDeleteAdvancedTimestamp = 0L; // for connected subscriptions, message expiry will be checked if the backlog is greater than this threshold private static final int MINIMUM_BACKLOG_FOR_EXPIRY_CHECK = 1000; @@ -139,7 +139,7 @@ static boolean isCursorFromReplicatedSubscription(ManagedCursor cursor) { } public PersistentSubscription(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, - boolean replicated) { + boolean replicated) { this(topic, subscriptionName, cursor, replicated, Collections.emptyMap()); } @@ -165,7 +165,7 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma public void updateLastMarkDeleteAdvancedTimestamp() { this.lastMarkDeleteAdvancedTimestamp = - Math.max(this.lastMarkDeleteAdvancedTimestamp, System.currentTimeMillis()); + Math.max(this.lastMarkDeleteAdvancedTimestamp, System.currentTimeMillis()); } @Override @@ -229,9 +229,9 @@ public CompletableFuture addConsumer(Consumer consumer) { previousDispatcher = dispatcher; dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer( - cursor, SubType.Exclusive, 0, topic, this) + cursor, SubType.Exclusive, 0, topic, this) : new PersistentDispatcherSingleActiveConsumer( - cursor, SubType.Exclusive, 0, topic, this); + cursor, SubType.Exclusive, 0, topic, this); } break; case Shared: @@ -239,7 +239,7 @@ public CompletableFuture addConsumer(Consumer consumer) { previousDispatcher = dispatcher; dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherMultipleConsumers( - topic, cursor, this) + topic, cursor, this) : new PersistentDispatcherMultipleConsumers(topic, cursor, this); } break; @@ -256,7 +256,7 @@ public CompletableFuture addConsumer(Consumer consumer) { previousDispatcher = dispatcher; dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer( - cursor, SubType.Failover, partitionIndex, topic, this) : + cursor, SubType.Failover, partitionIndex, topic, this) : new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, partitionIndex, topic, this); } @@ -265,7 +265,7 @@ public CompletableFuture addConsumer(Consumer consumer) { KeySharedMeta ksm = consumer.getKeySharedMeta(); if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared || !((PersistentStickyKeyDispatcherMultipleConsumers) dispatcher) - .hasSameKeySharedPolicy(ksm)) { + .hasSameKeySharedPolicy(ksm)) { previousDispatcher = dispatcher; dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this, topic.getBrokerService().getPulsar().getConfiguration(), ksm); @@ -337,7 +337,7 @@ public synchronized void removeConsumer(Consumer consumer, boolean isResetCursor // when topic closes: it iterates through concurrent-subscription map to close each subscription. so, // topic.remove again try to access same map which creates deadlock. so, execute it in different thread. - topic.getBrokerService().pulsar().getExecutor().execute(() ->{ + topic.getBrokerService().pulsar().getExecutor().execute(() -> { topic.removeSubscription(subName); // Also need remove the cursor here, otherwise the data deletion will not work well. // Because data deletion depends on the mark delete position of all cursors. @@ -412,7 +412,7 @@ public void acknowledgeMessage(List positions, AckType ackType, Map analyzeBacklog(Optional int batchSize = configuration.getDispatcherMaxReadBatchSize(); AtomicReference firstPosition = new AtomicReference<>(); AtomicReference lastPosition = new AtomicReference<>(); - return cursor.scan(position, new Predicate() { - @Override - public boolean apply(Entry entry) { - if (log.isDebugEnabled()) { - log.debug("found {}", entry); - } - Position entryPosition = entry.getPosition(); - firstPosition.compareAndSet(null, entryPosition); - lastPosition.set(entryPosition); - ByteBuf metadataAndPayload = entry.getDataBuffer(); - MessageMetadata messageMetadata = Commands.peekMessageMetadata(metadataAndPayload, "", -1); - int numMessages = 1; - if (messageMetadata.hasNumMessagesInBatch()) { - numMessages = messageMetadata.getNumMessagesInBatch(); - } - EntryFilter.FilterResult filterResult = entryFilterSupport - .runFiltersForEntry(entry, messageMetadata, null); - - if (filterResult == null) { - filterResult = EntryFilter.FilterResult.ACCEPT; - } - switch (filterResult) { - case REJECT: - rejected.incrementAndGet(); - rejectedMessages.addAndGet(numMessages); - break; - case RESCHEDULE: - rescheduled.incrementAndGet(); - rescheduledMessages.addAndGet(numMessages); - break; - default: - accepted.incrementAndGet(); - acceptedMessages.addAndGet(numMessages); - break; - } - long num = entries.incrementAndGet(); - messages.addAndGet(numMessages); - - if (num % 1000 == 0) { - long end = System.currentTimeMillis(); - log.info( - "[{}][{}] scan running since {} ms - scanned {} entries", - topicName, subName, end - start, num); - } + final Predicate condition = entry -> { + if (log.isDebugEnabled()) { + log.debug("found {}", entry); + } + Position entryPosition = entry.getPosition(); + firstPosition.compareAndSet(null, entryPosition); + lastPosition.set(entryPosition); + ByteBuf metadataAndPayload = entry.getDataBuffer(); + MessageMetadata messageMetadata = Commands.peekMessageMetadata(metadataAndPayload, "", -1); + int numMessages = 1; + if (messageMetadata.hasNumMessagesInBatch()) { + numMessages = messageMetadata.getNumMessagesInBatch(); + } + EntryFilter.FilterResult filterResult = entryFilterSupport + .runFiltersForEntry(entry, messageMetadata, null); - return true; + if (filterResult == null) { + filterResult = EntryFilter.FilterResult.ACCEPT; } - }, batchSize, maxEntries, timeOutMs).thenApply((ScanOutcome outcome) -> { + switch (filterResult) { + case REJECT: + rejected.incrementAndGet(); + rejectedMessages.addAndGet(numMessages); + break; + case RESCHEDULE: + rescheduled.incrementAndGet(); + rescheduledMessages.addAndGet(numMessages); + break; + default: + accepted.incrementAndGet(); + acceptedMessages.addAndGet(numMessages); + break; + } + long num = entries.incrementAndGet(); + messages.addAndGet(numMessages); + + if (num % 1000 == 0) { + long end = System.currentTimeMillis(); + log.info( + "[{}][{}] scan running since {} ms - scanned {} entries", + topicName, subName, end - start, num); + } + + return true; + }; + return cursor.scan( + position, + condition, + batchSize, + maxEntries, + timeOutMs + ).thenApply((ScanOutcome outcome) -> { long end = System.currentTimeMillis(); AnalyzeBacklogResult result = new AnalyzeBacklogResult(); result.setFirstPosition(firstPosition.get()); @@ -928,6 +932,7 @@ public CompletableFuture delete() { /** * Forcefully close all consumers and deletes the subscription. + * * @return */ @Override @@ -938,9 +943,10 @@ public CompletableFuture deleteForcefully() { /** * Delete the subscription by closing and deleting its managed cursor. Handle unsubscribe call from admin layer. * - * @param closeIfConsumersConnected - * Flag indicate whether explicitly close connected consumers before trying to delete subscription. If - * any consumer is connected to it and if this flag is disable then this operation fails. + * @param closeIfConsumersConnected Flag indicate whether explicitly close connected consumers before trying to + * delete subscription. If + * any consumer is connected to it and if this flag is disable then this operation + * fails. * @return CompletableFuture indicating the completion of delete operation */ private CompletableFuture delete(boolean closeIfConsumersConnected) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 49f2ac2f548cf..a3193935e5325 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled; import static org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic; @@ -26,8 +25,6 @@ import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.carrotsearch.hppc.ObjectObjectHashMap; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; @@ -35,8 +32,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CancellationException; @@ -73,6 +72,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ShadowManagedLedgerImpl; import org.apache.bookkeeper.net.BookieId; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -177,6 +177,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private final ConcurrentOpenHashMap shadowReplicators; @Getter private volatile List shadowTopics; + private final TopicName shadowSourceTopic; static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; private static final String TOPIC_EPOCH_PROPERTY_NAME = "pulsar.topic.epoch"; @@ -225,7 +226,7 @@ protected TopicStatsHelper initialValue() { protected final TransactionBuffer transactionBuffer; // Record the last time a data message (ie: not an internal Pulsar marker) is published on the topic - private long lastDataMessagePublishedTimestamp = 0; + private volatile long lastDataMessagePublishedTimestamp = 0; private static class TopicStatsHelper { public double averageMsgSize; @@ -303,6 +304,11 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS this.transactionBuffer = new TransactionBufferDisable(); } transactionBuffer.syncMaxReadPositionForNormalPublish((PositionImpl) ledger.getLastConfirmedEntry()); + if (ledger instanceof ShadowManagedLedgerImpl) { + shadowSourceTopic = ((ShadowManagedLedgerImpl) ledger).getShadowSource(); + } else { + shadowSourceTopic = null; + } } @Override @@ -382,6 +388,7 @@ public CompletableFuture initialize() { } else { this.transactionBuffer = new TransactionBufferDisable(); } + shadowSourceTopic = null; } private void initializeDispatchRateLimiterIfNeeded() { @@ -401,7 +408,7 @@ public AtomicLong getPendingWriteOps() { private PersistentSubscription createPersistentSubscription(String subscriptionName, ManagedCursor cursor, boolean replicated, Map subscriptionProperties) { - checkNotNull(compactedTopic); + Objects.requireNonNull(compactedTopic); if (isCompactionSubscription(subscriptionName)) { return new CompactorSubscription(this, compactedTopic, subscriptionName, cursor); } else { @@ -553,7 +560,7 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) // close all producers CompletableFuture disconnectProducersFuture; if (producers.size() > 0) { - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); producers.forEach((__, producer) -> futures.add(producer.disconnect())); disconnectProducersFuture = FutureUtil.waitForAll(futures); } else { @@ -669,14 +676,14 @@ public CompletableFuture startReplProducers() { } public CompletableFuture stopReplProducers() { - List> closeFutures = Lists.newArrayList(); + List> closeFutures = new ArrayList<>(); replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect())); return FutureUtil.waitForAll(closeFutures); } private synchronized CompletableFuture closeReplProducersIfNoBacklog() { - List> closeFutures = Lists.newArrayList(); + List> closeFutures = new ArrayList<>(); replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect(true))); shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect(true))); return FutureUtil.waitForAll(closeFutures); @@ -1162,7 +1169,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, fenceTopicToCloseOrDelete(); // Avoid clients reconnections while deleting CompletableFuture closeClientFuture = new CompletableFuture<>(); if (closeIfClientsConnected) { - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); shadowReplicators.forEach((__, replicator) -> futures.add(replicator.disconnect())); producers.values().forEach(producer -> futures.add(producer.disconnect())); @@ -1286,7 +1293,7 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect lock.writeLock().unlock(); } - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); futures.add(transactionBuffer.closeAsync()); replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); @@ -1423,7 +1430,7 @@ public CompletableFuture checkReplication() { return deleteForcefully(); } - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); // Check for missing replicators for (String cluster : configuredClusters) { @@ -1461,7 +1468,7 @@ private CompletableFuture checkShadowReplication() { if (log.isDebugEnabled()) { log.debug("[{}] Checking shadow replication status, shadowTopics={}", topic, configuredShadowTopics); } - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); // Check for missing replicators for (String shadowTopic : configuredShadowTopics) { @@ -2142,7 +2149,7 @@ public CompletableFuture getInternalStats(boolean stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString(); stats.state = ml.getState().toString(); - stats.ledgers = Lists.newArrayList(); + stats.ledgers = new ArrayList<>(); Set> futures = Sets.newConcurrentHashSet(); CompletableFuture> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync(); @@ -2197,7 +2204,7 @@ public CompletableFuture getInternalStats(boolean stats.compactedLedger = info; - stats.cursors = Maps.newTreeMap(); + stats.cursors = new HashMap<>(); ml.getCursors().forEach(c -> { ManagedCursorImpl cursor = (ManagedCursorImpl) c; CursorStats cs = new CursorStats(); @@ -2211,6 +2218,7 @@ public CompletableFuture getInternalStats(boolean cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); cs.state = cursor.getState(); + cs.active = cursor.isActive(); cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage(); cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange(); cs.properties = cursor.getProperties(); @@ -2805,7 +2813,7 @@ public boolean isOldestMessageExpired(ManagedCursor cursor, int messageTTLInSeco */ public CompletableFuture clearBacklog() { log.info("[{}] Clearing backlog on all cursors in the topic.", topic); - List> futures = Lists.newArrayList(); + List> futures = new ArrayList<>(); List cursors = getSubscriptions().keys(); cursors.addAll(getReplicators().keys()); cursors.addAll(getShadowReplicators().keys()); @@ -3330,4 +3338,8 @@ private CompletableFuture transactionBufferCleanupAndClose() { public long getLastDataMessagePublishedTimestamp() { return lastDataMessagePublishedTimestamp; } + + public Optional getShadowSourceTopic() { + return Optional.ofNullable(shadowSourceTopic); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index a77ac76be8ff4..cf46fac4a24a8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -59,7 +59,7 @@ public class ReplicatedSubscriptionsController implements AutoCloseable, Topic.P private final String localCluster; // The timestamp of when the last snapshot was initiated - private long lastCompletedSnapshotStartTime = 0; + private volatile long lastCompletedSnapshotStartTime = 0; private String lastCompletedSnapshotId; @@ -202,7 +202,8 @@ private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) { private void startNewSnapshot() { cleanupTimedOutSnapshots(); - if (topic.getLastDataMessagePublishedTimestamp() < lastCompletedSnapshotStartTime) { + if (topic.getLastDataMessagePublishedTimestamp() < lastCompletedSnapshotStartTime + || topic.getLastDataMessagePublishedTimestamp() == 0) { // There was no message written since the last snapshot, we can skip creating a new snapshot if (log.isDebugEnabled()) { log.debug("[{}] There is no new data in topic. Skipping snapshot creation.", topic.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index b62335d9ab435..9a78de6e30ba4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.schema; -import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Lists.newArrayList; import static com.google.protobuf.ByteString.copyFrom; import static java.util.Objects.isNull; @@ -463,12 +462,15 @@ private CompletableFuture updateSchemaLocator( .setHash(copyFrom(hash)) .build(); + final ArrayList indexList = new ArrayList<>(); + indexList.addAll(locator.getIndexList()); + indexList.add(info); return updateSchemaLocator(getSchemaPath(schemaId), - SchemaStorageFormat.SchemaLocator.newBuilder() - .setInfo(info) - .addAllIndex( - concat(locator.getIndexList(), newArrayList(info)) - ).build(), locatorEntry.version + SchemaStorageFormat.SchemaLocator.newBuilder() + .setInfo(info) + .addAllIndex(indexList) + .build() + , locatorEntry.version ).thenApply(ignore -> nextVersion); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java index e78477a9aa080..abf74cdc4503a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.service.schema; -import com.google.common.collect.Maps; +import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; @@ -34,7 +34,7 @@ public interface SchemaRegistryService extends SchemaRegistry { long NO_SCHEMA_VERSION = -1L; static Map getCheckers(Set checkerClasses) throws Exception { - Map checkers = Maps.newHashMap(); + Map checkers = new HashMap<>(); for (String className : checkerClasses) { SchemaCompatibilityCheck schemaCompatibilityCheck = Reflections.createInstance(className, SchemaCompatibilityCheck.class, Thread.currentThread().getContextClassLoader()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java index 5dd4c0b0402c9..9e71fb7d46bfe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java @@ -18,12 +18,12 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Lists; import io.netty.buffer.PoolArenaMetric; import io.netty.buffer.PoolChunkListMetric; import io.netty.buffer.PoolChunkMetric; import io.netty.buffer.PoolSubpageMetric; import io.netty.buffer.PooledByteBufAllocator; +import java.util.ArrayList; import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl; import org.apache.pulsar.common.stats.AllocatorStats; @@ -99,7 +99,7 @@ private static PoolChunkListStats newPoolChunkListStats(PoolChunkListMetric m) { PoolChunkListStats stats = new PoolChunkListStats(); stats.minUsage = m.minUsage(); stats.maxUsage = m.maxUsage(); - stats.chunks = Lists.newArrayList(); + stats.chunks = new ArrayList<>(); m.forEach(chunk -> stats.chunks.add(newPoolChunkStats(chunk))); return stats; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BookieClientStatsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BookieClientStatsGenerator.java index d34c430f19c59..c92693c0df4d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BookieClientStatsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BookieClientStatsGenerator.java @@ -18,8 +18,9 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Maps; +import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; import org.apache.bookkeeper.mledger.proto.PendingBookieOpsStats; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -35,7 +36,7 @@ public class BookieClientStatsGenerator { public BookieClientStatsGenerator(PulsarService pulsar) { this.pulsar = pulsar; - this.nsBookieClientStatsMap = Maps.newTreeMap(); + this.nsBookieClientStatsMap = new TreeMap<>(); } public static Map> generate(PulsarService pulsar) throws Exception { @@ -60,7 +61,7 @@ private Map> generate() throws Except private void put(TopicName topicName, PendingBookieOpsStats bookieOpsStats) { String namespace = topicName.getNamespace(); if (!nsBookieClientStatsMap.containsKey(namespace)) { - Map destBookieClientStatsMap = Maps.newTreeMap(); + Map destBookieClientStatsMap = new HashMap<>(); destBookieClientStatsMap.put(topicName.toString(), bookieOpsStats); nsBookieClientStatsMap.put(namespace, destBookieClientStatsMap); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java index 4fd9e35dd7be3..ac131121d0b79 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Maps; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -82,7 +82,7 @@ Metrics getConnectionMetrics() { } Map getDimensionMap(String metricsName) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("broker", brokerName); dimensionMap.put("cluster", localCluster); dimensionMap.put("metric", metricsName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MBeanStatsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MBeanStatsGenerator.java index 6b5d63bee1b70..905db1c1e64ed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MBeanStatsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MBeanStatsGenerator.java @@ -18,10 +18,10 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Maps; import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,7 +108,7 @@ private Metrics convert(ObjectInstance instance) { * @return */ private Metrics createMetricsByDimension(ObjectName objectName) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("MBean", objectName.toString()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MetricsGenerator.java index 5239cfb9acc8d..d8c518e9a5d4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/MetricsGenerator.java @@ -18,9 +18,9 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.pulsar.broker.PulsarService; @@ -61,7 +61,7 @@ private Collection merge(List metricsCollection) { // map by dimension map -> metrics // since dimension map is unique - Map, Metrics> mergedMetrics = Maps.newHashMap(); + Map, Metrics> mergedMetrics = new HashMap<>(); for (Metrics metrics : metricsCollection) { Map dimensionKey = metrics.getDimensions(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java index 09db78d1edd54..d8047f1e13ea8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java @@ -19,8 +19,8 @@ package org.apache.pulsar.broker.stats; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; -import com.google.common.collect.Maps; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import org.apache.pulsar.common.stats.Metrics; @@ -85,7 +85,7 @@ public void reset() { public Metrics add(String namespace) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("namespace", namespace); Metrics dMetrics = Metrics.create(dimensionMap); dMetrics.put("brk_in_rate", msgRateIn); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ReplicationMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ReplicationMetrics.java index 28f36190428a5..0046218d7c2c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ReplicationMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ReplicationMetrics.java @@ -18,9 +18,9 @@ */ package org.apache.pulsar.broker.stats; -import com.google.common.collect.Maps; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; +import java.util.HashMap; import java.util.Map; import org.apache.pulsar.common.stats.Metrics; @@ -70,7 +70,7 @@ public void recycle() { public Metrics add(String namespace, String local, String remote) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("namespace", namespace); dimensionMap.put("from_cluster", local); dimensionMap.put("to_cluster", remote); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java index 610d22c54d8e9..e7e19c8ce3ba3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java @@ -18,8 +18,7 @@ */ package org.apache.pulsar.broker.stats.metrics; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -188,7 +187,7 @@ protected String parseNamespaceFromLedgerName(String ledgerName) { * @return */ protected Metrics createMetricsByDimension(String namespace) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("namespace", namespace); @@ -204,7 +203,7 @@ protected Metrics createMetricsByDimension(String namespace) { * @return */ protected Metrics createMetricsByDimension(String namespace, String fromClusterName, String toClusterName) { - Map dimensionMap = Maps.newHashMap(); + Map dimensionMap = new HashMap<>(); dimensionMap.put("namespace", namespace); dimensionMap.put("from_cluster", fromClusterName); @@ -214,7 +213,7 @@ protected Metrics createMetricsByDimension(String namespace, String fromClusterN } protected void populateAggregationMap(Map> map, String mkey, double value) { - map.computeIfAbsent(mkey, __ -> Lists.newArrayList()).add(value); + map.computeIfAbsent(mkey, __ -> new ArrayList<>()).add(value); } protected void populateAggregationMapWithSum(Map map, String mkey, double value) { @@ -238,11 +237,11 @@ protected void populateMaxMap(Map map, String mkey, long value) { */ protected void populateDimensionMap(Map> ledgersByDimensionMap, Metrics metrics, ManagedLedgerImpl ledger) { - ledgersByDimensionMap.computeIfAbsent(metrics, __ -> Lists.newArrayList()).add(ledger); + ledgersByDimensionMap.computeIfAbsent(metrics, __ -> new ArrayList<>()).add(ledger); } protected void populateDimensionMap(Map> topicsStatsByDimensionMap, Metrics metrics, TopicStats destStats) { - topicsStatsByDimensionMap.computeIfAbsent(metrics, __ -> Lists.newArrayList()).add(destStats); + topicsStatsByDimensionMap.computeIfAbsent(metrics, __ -> new ArrayList<>()).add(destStats); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java index 17fbc270e0d43..1f3b58a2590f4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.stats.metrics; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -38,8 +38,8 @@ public class ManagedCursorMetrics extends AbstractMetrics { public ManagedCursorMetrics(PulsarService pulsar) { super(pulsar); - this.metricsCollection = Lists.newArrayList(); - this.dimensionMap = Maps.newHashMap(); + this.metricsCollection = new ArrayList<>(); + this.dimensionMap = new HashMap<>(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java index 1f4181f887fdd..7bbef5379cbae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java @@ -18,11 +18,11 @@ */ package org.apache.pulsar.broker.stats.metrics; -import com.google.common.collect.Lists; import io.netty.buffer.PoolArenaMetric; import io.netty.buffer.PoolChunkListMetric; import io.netty.buffer.PoolChunkMetric; import io.netty.buffer.PooledByteBufAllocator; +import java.util.ArrayList; import java.util.List; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl; @@ -34,7 +34,7 @@ public class ManagedLedgerCacheMetrics extends AbstractMetrics { private List metrics; public ManagedLedgerCacheMetrics(PulsarService pulsar) { super(pulsar); - this.metrics = Lists.newArrayList(); + this.metrics = new ArrayList<>(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java index 8a9dd4a3da6cd..a8edbf3dd3e01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.stats.metrics; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -50,9 +50,9 @@ public class ManagedLedgerMetrics extends AbstractMetrics { public ManagedLedgerMetrics(PulsarService pulsar) { super(pulsar); - this.metricsCollection = Lists.newArrayList(); - this.ledgersByDimensionMap = Maps.newHashMap(); - this.tempAggregatedMetricsMap = Maps.newHashMap(); + this.metricsCollection = new ArrayList<>(); + this.ledgersByDimensionMap = new HashMap<>(); + this.tempAggregatedMetricsMap = new HashMap<>(); this.statsPeriodSeconds = ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()) .getConfig().getStatsPeriodSeconds(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClientBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClientBase.java index cb45077341d22..98465d40f5730 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClientBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/SystemTopicClientBase.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.systopic; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -86,9 +85,9 @@ public CompletableFuture> newWriterAsync() { @Override public CompletableFuture closeAsync() { List> futures = new ArrayList<>(); - List> tempWriters = Lists.newArrayList(writers); + List> tempWriters = new ArrayList<>(writers); tempWriters.forEach(writer -> futures.add(writer.closeAsync())); - List> tempReaders = Lists.newArrayList(readers); + List> tempReaders = new ArrayList<>(readers); tempReaders.forEach(reader -> futures.add(reader.closeAsync())); writers.clear(); readers.clear(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 3ff4ed8e76808..8b1e0a4e94123 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -61,6 +61,7 @@ import org.apache.pulsar.transaction.coordinator.impl.TxnBatchedPositionImpl; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriter; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; +import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterMetricsStats; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; import org.slf4j.Logger; @@ -119,7 +120,7 @@ public class MLPendingAckStore implements PendingAckStore { public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, ManagedCursor subManagedCursor, long transactionPendingAckLogIndexMinLag, TxnLogBufferedWriterConfig bufferedWriterConfig, - Timer timer) { + Timer timer, TxnLogBufferedWriterMetricsStats bufferedWriterMetrics) { this.managedLedger = managedLedger; this.cursor = cursor; this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); @@ -132,7 +133,8 @@ public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, this.bufferedWriter = new TxnLogBufferedWriter(managedLedger, ((ManagedLedgerImpl) managedLedger).getExecutor(), timer, PendingAckLogSerializer.INSTANCE, bufferedWriterConfig.getBatchedWriteMaxRecords(), bufferedWriterConfig.getBatchedWriteMaxSize(), - bufferedWriterConfig.getBatchedWriteMaxDelayInMillis(), bufferedWriterConfig.isBatchEnabled()); + bufferedWriterConfig.getBatchedWriteMaxDelayInMillis(), bufferedWriterConfig.isBatchEnabled(), + bufferedWriterMetrics); this.batchedPendingAckLogsWaitingForHandle = new ArrayList<>(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index fdff9f59146d4..bf2771abaa65a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.transaction.pendingack.impl; import io.netty.util.Timer; +import io.prometheus.client.CollectorRegistry; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -34,7 +35,9 @@ import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; +import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterMetricsStats; /** @@ -43,6 +46,21 @@ @Slf4j public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + private static volatile TxnLogBufferedWriterMetricsStats bufferedWriterMetrics = + DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; + + public static void initBufferedWriterMetrics(String brokerAdvertisedAddress){ + if (bufferedWriterMetrics != DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS){ + return; + } + synchronized (MLPendingAckStoreProvider.class){ + if (bufferedWriterMetrics != DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS){ + return; + } + bufferedWriterMetrics = new MLTxnPendingAckLogBufferedWriterMetrics(brokerAdvertisedAddress); + } + } + @Override public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); @@ -105,7 +123,7 @@ public void openCursorComplete(ManagedCursor cursor, Object ctx) { .getConfiguration() .getTransactionPendingAckLogIndexMinLag(), txnLogBufferedWriterConfig, - brokerClientSharedTimer)); + brokerClientSharedTimer, bufferedWriterMetrics)); if (log.isDebugEnabled()) { log.debug("{},{} open MLPendingAckStore cursor success", originPersistentTopic.getName(), @@ -151,4 +169,14 @@ public CompletableFuture checkInitializedBefore(PersistentSubscription return originPersistentTopic.getBrokerService().getManagedLedgerFactory() .asyncExists(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding()); } + + private static class MLTxnPendingAckLogBufferedWriterMetrics extends TxnLogBufferedWriterMetricsStats{ + + private MLTxnPendingAckLogBufferedWriterMetrics(String brokerAdvertisedAddress) { + super("pulsar_txn_pending_ack_store", + new String[]{"broker"}, + new String[]{brokerAdvertisedAddress}, + CollectorRegistry.defaultRegistry); + } + } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java index 4a8274389eb62..b6ae55b00b84b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java @@ -30,16 +30,16 @@ public class CompactionRecord { 200_000, 1000_000 }; @Getter - private long lastCompactionRemovedEventCount = 0L; + private volatile long lastCompactionRemovedEventCount = 0L; @Getter - private long lastCompactionSucceedTimestamp = 0L; + private volatile long lastCompactionSucceedTimestamp = 0L; @Getter - private long lastCompactionFailedTimestamp = 0L; + private volatile long lastCompactionFailedTimestamp = 0L; @Getter - private long lastCompactionDurationTimeInMills = 0L; + private volatile long lastCompactionDurationTimeInMills = 0L; private LongAdder lastCompactionRemovedEventCountOp = new LongAdder(); - private long lastCompactionStartTimeOp; + private volatile long lastCompactionStartTimeOp; private final LongAdder compactionRemovedEventCount = new LongAdder(); private final LongAdder compactionSucceedCount = new LongAdder(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 4cb1fd347df13..91c079b27becc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -31,13 +31,13 @@ import java.util.concurrent.ScheduledExecutorService; import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.pulsar.broker.BookKeeperClientFactory; import org.apache.pulsar.broker.BookKeeperClientFactoryImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.ServiceConfigurationUtils; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; @@ -65,6 +65,45 @@ private static class Arguments { private boolean generateDocs = false; } + public static PulsarClient createClient(ServiceConfiguration brokerConfig) throws PulsarClientException { + ClientBuilder clientBuilder = PulsarClient.builder() + .memoryLimit(0, SizeUnit.BYTES); + + // Apply all arbitrary configuration. This must be called before setting any fields annotated as + // @Secret on the ClientConfigurationData object because of the way they are serialized. + // See https://github.com/apache/pulsar/issues/8509 for more information. + clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(brokerConfig.getProperties(), "brokerClient_")); + + if (isNotBlank(brokerConfig.getBrokerClientAuthenticationPlugin())) { + clientBuilder.authentication(brokerConfig.getBrokerClientAuthenticationPlugin(), + brokerConfig.getBrokerClientAuthenticationParameters()); + } + + AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar+ssl"); + if (internalListener.getBrokerServiceUrlTls() != null && brokerConfig.isBrokerClientTlsEnabled()) { + clientBuilder.serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) + .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); + if (brokerConfig.isBrokerClientTlsEnabledWithKeyStore()) { + clientBuilder.useKeyStoreTls(true) + .tlsKeyStoreType(brokerConfig.getBrokerClientTlsKeyStoreType()) + .tlsKeyStorePath(brokerConfig.getBrokerClientTlsKeyStore()) + .tlsKeyStorePassword(brokerConfig.getBrokerClientTlsKeyStorePassword()) + .tlsTrustStoreType(brokerConfig.getBrokerClientTlsTrustStoreType()) + .tlsTrustStorePath(brokerConfig.getBrokerClientTlsTrustStore()) + .tlsTrustStorePassword(brokerConfig.getBrokerClientTlsTrustStorePassword()); + } else { + clientBuilder.tlsTrustCertsFilePath(brokerConfig.getBrokerClientTrustCertsFilePath()) + .tlsKeyFilePath(brokerConfig.getBrokerClientKeyFilePath()) + .tlsCertificateFilePath(brokerConfig.getBrokerClientCertificateFilePath()); + } + } else { + internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar"); + clientBuilder.serviceUrl(internalListener.getBrokerServiceUrl().toString()); + } + + return clientBuilder.build(); + } + public static void main(String[] args) throws Exception { Arguments arguments = new Arguments(); JCommander jcommander = new JCommander(arguments); @@ -105,40 +144,10 @@ public static void main(String[] args) throws Exception { ); } - ClientBuilder clientBuilder = PulsarClient.builder() - .memoryLimit(0, SizeUnit.BYTES); - - // Apply all arbitrary configuration. This must be called before setting any fields annotated as - // @Secret on the ClientConfigurationData object because of the way they are serialized. - // See https://github.com/apache/pulsar/issues/8509 for more information. - clientBuilder.loadConf(PropertiesUtils.filterAndMapProperties(brokerConfig.getProperties(), "brokerClient_")); - - if (isNotBlank(brokerConfig.getBrokerClientAuthenticationPlugin())) { - clientBuilder.authentication(brokerConfig.getBrokerClientAuthenticationPlugin(), - brokerConfig.getBrokerClientAuthenticationParameters()); - } - - AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar+ssl"); - if (internalListener.getBrokerServiceUrlTls() != null) { - log.info("Found a TLS-based advertised listener in configuration file. \n" - + "Will connect pulsar use TLS."); - clientBuilder - .serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) - .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()) - .tlsTrustCertsFilePath(brokerConfig.getTlsCertificateFilePath()); - - } else { - internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar"); - clientBuilder.serviceUrl(internalListener.getBrokerServiceUrl().toString()); - } - @Cleanup(value = "shutdownNow") ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build()); - @Cleanup(value = "shutdownNow") - OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build(); - @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(brokerConfig.getMetadataStoreUrl(), MetadataStoreConfig.builder() @@ -157,7 +166,7 @@ public static void main(String[] args) throws Exception { BookKeeper bk = bkClientFactory.create(brokerConfig, store, eventLoopGroup, Optional.empty(), null); @Cleanup - PulsarClient pulsar = clientBuilder.build(); + PulsarClient pulsar = createClient(brokerConfig); Compactor compactor = new TwoPhaseCompactor(brokerConfig, pulsar, bk, scheduler); long ledgerId = compactor.compact(arguments.topic).get(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java index 3f5df60faea69..c784ff84408d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java @@ -21,9 +21,9 @@ import com.beust.jcommander.Parameters; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.BaseGenerateDocumentation; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.BaseGenerateDocumentation; import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; @Data diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java index 224060c9d912e..7d7a3ebd2621d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java @@ -34,14 +34,47 @@ public static String newUniqueName(String prefix) { * Creates a Mockito spy directly without an intermediate instance to spy. * This is to address flaky test issue where a spy created with a given instance fails with * {@link org.mockito.exceptions.misusing.WrongTypeOfReturnValue} exception. + * The spy is stub-only which does not record method invocations. * * @param classToSpy the class to spy * @param args the constructor arguments to use when creating the spy instance * @return a spy of the provided class created with given constructor arguments */ public static T spyWithClassAndConstructorArgs(Class classToSpy, Object... args) { + return Mockito.mock(classToSpy, Mockito.withSettings() + .useConstructor(args) + .defaultAnswer(Mockito.CALLS_REAL_METHODS) + .stubOnly()); + } + + /** + * Creates a Mockito spy directly without an intermediate instance to spy. + * This is to address flaky test issue where a spy created with a given instance fails with + * {@link org.mockito.exceptions.misusing.WrongTypeOfReturnValue} exception. + * The spy records method invocations. + * + * @param classToSpy the class to spy + * @param args the constructor arguments to use when creating the spy instance + * @return a spy of the provided class created with given constructor arguments + */ + public static T spyWithClassAndConstructorArgsRecordingInvocations(Class classToSpy, Object... args) { return Mockito.mock(classToSpy, Mockito.withSettings() .useConstructor(args) .defaultAnswer(Mockito.CALLS_REAL_METHODS)); } + + /** + * Create a Mockito spy that is stub-only which does not record method invocations, + * thus saving memory but disallowing verification of invocations. + * + * @param object to spy on + * @return a spy of the real object + * @param type of object + */ + public static T spyWithoutRecordingInvocations(T object) { + return Mockito.mock((Class) object.getClass(), Mockito.withSettings() + .spiedInstance(object) + .defaultAnswer(Mockito.CALLS_REAL_METHODS) + .stubOnly()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 888106ead3114..b71ad79c48728 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -363,7 +364,7 @@ public void nonPersistentTopics() throws Exception { publishMessagesOnTopic(nonPersistentTopicName, 10, 0); NonPersistentTopicStats topicStats = (NonPersistentTopicStats) admin.topics().getStats(nonPersistentTopicName); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(topicStats.getSubscriptions().keySet(), Set.of("my-sub")); assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgDropRate(), 0); assertEquals(topicStats.getPublishers().size(), 0); @@ -372,7 +373,7 @@ public void nonPersistentTopics() throws Exception { pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort().get()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(nonPersistentTopicName, false); - assertEquals(internalStats.cursors.keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(internalStats.cursors.keySet(), Set.of("my-sub")); consumer.close(); topicStats = (NonPersistentTopicStats) admin.topics().getStats(nonPersistentTopicName); @@ -545,7 +546,7 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Shared).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0); @@ -684,21 +685,21 @@ public void testPeerCluster() throws Exception { admin.clusters().createCluster("us-east2", ClusterData.builder().serviceUrl("http://broker.messaging.east2.example.com:8080").build()); - admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west2"))); - assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), Lists.newArrayList("us-west2")); + admin.clusters().updatePeerClusterNames("us-west1", new LinkedHashSet<>(List.of("us-west2"))); + assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), Set.of("us-west2")); assertNull(admin.clusters().getCluster("us-west2").getPeerClusterNames()); // update cluster with duplicate peer-clusters in the list - admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet( - Lists.newArrayList("us-west2", "us-east1", "us-west2", "us-east1", "us-west2", "us-east1"))); + admin.clusters().updatePeerClusterNames("us-west1", + new LinkedHashSet<>(List.of("us-west2", "us-east1", "us-west2", "us-east1", "us-west2", "us-east1"))); assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), - Lists.newArrayList("us-west2", "us-east1")); + List.of("us-west2", "us-east1")); admin.clusters().updatePeerClusterNames("us-west1", null); assertNull(admin.clusters().getCluster("us-west1").getPeerClusterNames()); // Check name validation try { admin.clusters().updatePeerClusterNames("us-west1", - Sets.newLinkedHashSet(Lists.newArrayList("invalid-cluster"))); + new LinkedHashSet<>(List.of("invalid-cluster"))); fail("should have failed"); } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); @@ -706,7 +707,7 @@ public void testPeerCluster() throws Exception { // Cluster itself can't be part of peer-list try { - admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west1"))); + admin.clusters().updatePeerClusterNames("us-west1", new LinkedHashSet<>(List.of("us-west1"))); fail("should have failed"); } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); @@ -737,7 +738,7 @@ public void testReplicationPeerCluster() throws Exception { List allClusters = admin.clusters().getClusters(); Collections.sort(allClusters); assertEquals(allClusters, - Lists.newArrayList("test", "us-east1", "us-east2", "us-west1", "us-west2", "us-west3", "us-west4")); + List.of("test", "us-east1", "us-east2", "us-west1", "us-west2", "us-west3", "us-west4")); final String property = "peer-prop"; Set allowedClusters = Set.of("us-west1", "us-west2", "us-west3", "us-west4", "us-east1", @@ -749,9 +750,9 @@ public void testReplicationPeerCluster() throws Exception { admin.namespaces().createNamespace(namespace); admin.clusters().updatePeerClusterNames("us-west1", - Sets.newLinkedHashSet(Lists.newArrayList("us-west2", "us-west3"))); + new LinkedHashSet<>(List.of("us-west2", "us-west3"))); assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), - Lists.newArrayList("us-west2", "us-west3")); + List.of("us-west2", "us-west3")); // (1) no conflicting peer Set clusterIds = Set.of("us-east1", "us-east2"); @@ -1241,7 +1242,7 @@ public void clustersList() throws PulsarAdminException { .serviceUrl("http://localhost:6650").build()); // Global cluster, if there, should be omitted from the results - assertEquals(admin.clusters().getClusters(), Lists.newArrayList(cluster)); + assertEquals(admin.clusters().getClusters(), List.of(cluster)); } /** * verifies cluster has been set before create topic @@ -1462,7 +1463,7 @@ public void testDeleteTenant() throws Exception { // create namespace String namespace = tenant + "/test-ns-1"; admin.namespaces().createNamespace(namespace, Set.of("test")); - assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList(namespace)); + assertEquals(admin.namespaces().getNamespaces(tenant), List.of(namespace)); // create topic String topic = namespace + "/test-topic-1"; @@ -1554,7 +1555,7 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { // create namespace String namespace = tenant + "/test-ns"; admin.namespaces().createNamespace(namespace, Set.of("test")); - assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList(namespace)); + assertEquals(admin.namespaces().getNamespaces(tenant), List.of(namespace)); // create topic String topic = namespace + "/test-topic"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 2246e184384e3..c75bb280faaa3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -244,14 +244,14 @@ public void clusters() throws Exception { ClusterData.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build()); // "test" cluster is part of config-default cluster and it's znode gets created when PulsarService creates // failure-domain znode of this default cluster - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("test", "usw")); assertEquals(admin.clusters().getCluster("test"), ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); admin.clusters().updateCluster("usw", ClusterData.builder().serviceUrl("http://new-broker.messaging.usw.example.com:8080").build()); - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("test", "usw")); assertEquals(admin.clusters().getCluster("usw"), ClusterData.builder().serviceUrl("http://new-broker.messaging.usw.example.com:8080").build()); @@ -260,7 +260,7 @@ public void clusters() throws Exception { .serviceUrl("http://new-broker.messaging.usw.example.com:8080") .serviceUrlTls("https://new-broker.messaging.usw.example.com:4443") .build()); - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("test", "usw")); assertEquals(admin.clusters().getCluster("usw"), ClusterData.builder() .serviceUrl("http://new-broker.messaging.usw.example.com:8080") @@ -269,7 +269,7 @@ public void clusters() throws Exception { admin.clusters().deleteCluster("usw"); Awaitility.await() - .untilAsserted(() -> assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test"))); + .untilAsserted(() -> assertEquals(admin.clusters().getClusters(), List.of("test"))); deleteNamespaceGraceFully("prop-xyz/ns1", false); admin.clusters().deleteCluster("test"); @@ -439,7 +439,7 @@ public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { NamespaceIsolationData.Builder nsRegexPolicy = NamespaceIsolationData.builder() .namespaces(Collections.singletonList("other/use/other.*")) - .primary(Lists.newArrayList("prod1-broker[45-46].messaging.use.example.com")) + .primary(List.of("prod1-broker[45-46].messaging.use.example.com")) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) .parameters(parameters) @@ -451,7 +451,7 @@ public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { //Ok } - nsRegexPolicy.primary(Lists.newArrayList("prod1-broker[45-46].messaging.use.example.com", + nsRegexPolicy.primary(List.of("prod1-broker[45-46].messaging.use.example.com", "prod1-broker[4-5].messaging.use.example.com")) .secondary(Collections.singletonList("prod1-broker[45-46].messaging.use.example.com")); try { @@ -714,7 +714,7 @@ public void properties() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), allowedClusters); admin.tenants().updateTenant("prop-xyz", tenantInfo); - assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz")); + assertEquals(admin.tenants().getTenants(), List.of("prop-xyz")); assertEquals(admin.tenants().getTenantInfo("prop-xyz"), tenantInfo); @@ -769,8 +769,8 @@ public void namespaces() throws Exception { // Ok } - assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns1", "prop-xyz/ns2")); - assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns1", "prop-xyz/ns2")); + assertEquals(admin.namespaces().getNamespaces("prop-xyz"), List.of("prop-xyz/ns1", "prop-xyz/ns2")); + assertEquals(admin.namespaces().getNamespaces("prop-xyz"), List.of("prop-xyz/ns1", "prop-xyz/ns2")); try { admin.namespaces().createNamespace("prop-xyz/ns4", Set.of("usc")); @@ -835,7 +835,7 @@ public void namespaces() throws Exception { assertTrue(i < 10); deleteNamespaceGraceFully("prop-xyz/ns1", false); - assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns2")); + assertEquals(admin.namespaces().getNamespaces("prop-xyz"), List.of("prop-xyz/ns2")); try { admin.namespaces().unload("prop-xyz/ns1"); @@ -862,7 +862,7 @@ public void persistentTopics(String topicName) throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0); assertEquals(admin.topics().getList("prop-xyz/ns1"), - Lists.newArrayList("persistent://prop-xyz/ns1/" + topicName)); + List.of("persistent://prop-xyz/ns1/" + topicName)); // create consumer and subscription @Cleanup @@ -873,12 +873,12 @@ public void persistentTopics(String topicName) throws Exception { Consumer consumer = client.newConsumer().topic(persistentTopicName).subscriptionName(subName) .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList(subName)); + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName)); publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 10); TopicStats topicStats = admin.topics().getStats(persistentTopicName); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList(subName))); + assertEquals(topicStats.getSubscriptions().keySet(), Set.of(subName)); assertEquals(topicStats.getSubscriptions().get(subName).getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); assertEquals(topicStats.getPublishers().size(), 0); @@ -886,7 +886,7 @@ public void persistentTopics(String topicName) throws Exception { pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort().get()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName, false); - assertEquals(internalStats.cursors.keySet(), new TreeSet<>(Lists.newArrayList(Codec.encode(subName)))); + assertEquals(internalStats.cursors.keySet(), Set.of(Codec.encode(subName))); List> messages = admin.topics().peekMessages(persistentTopicName, subName, 3); assertEquals(messages.size(), 3); @@ -968,7 +968,7 @@ public void partitionedTopics(String topicType, String topicName) throws Excepti admin.topics().createPartitionedTopic(partitionedTopicName, 4); assertEquals(admin.topics().getPartitionedTopicList(namespace), - Lists.newArrayList(partitionedTopicName)); + List.of(partitionedTopicName)); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4); @@ -1005,7 +1005,7 @@ public void partitionedTopics(String topicType, String topicName) throws Excepti Consumer consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(partitionedTopicName), List.of("my-sub")); try { if (isPersistent) { @@ -1034,7 +1034,7 @@ public void partitionedTopics(String topicType, String topicName) throws Excepti // TODO: for non-persistent topics, deleteSubscription might throw NotFoundException admin.topics().deleteSubscription(partitionedTopicName, "my-sub-1"); // TODO: for non-persistent topics, getSubscriptions will return a empty set - assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(partitionedTopicName), List.of("my-sub")); } Producer producer = client.newProducer(Schema.BYTES) @@ -1056,7 +1056,7 @@ public void partitionedTopics(String topicType, String topicName) throws Excepti topicStats = admin.topics().getPartitionedStats(partitionedTopicName,false); if (isPersistent) { // TODO: for non-persistent topics, the subscription doesn't exist - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(topicStats.getSubscriptions().keySet(), Set.of("my-sub")); assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgBacklog(), 10); } @@ -1160,7 +1160,7 @@ public void testGetPartitionedInternalInfo() throws Exception { assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), new ArrayList<>()); final String partitionedTopicName = "persistent://prop-xyz/ns1/" + partitionedTopic; admin.topics().createPartitionedTopic(partitionedTopicName, 2); - assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), Lists.newArrayList(partitionedTopicName)); + assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), List.of(partitionedTopicName)); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 2); String partitionTopic0 = partitionedTopicName + "-partition-0"; @@ -1223,7 +1223,7 @@ public void testGetPartitionedStatsInternal() throws Exception { assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), new ArrayList<>()); final String partitionedTopicName = "persistent://prop-xyz/ns1/" + partitionedTopic; admin.topics().createPartitionedTopic(partitionedTopicName, 2); - assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), Lists.newArrayList(partitionedTopicName)); + assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), List.of(partitionedTopicName)); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 2); // create consumer and subscription @@ -1238,10 +1238,10 @@ public void testGetPartitionedStatsInternal() throws Exception { Thread.sleep(1000); PersistentTopicInternalStats internalStats0 = admin.topics().getInternalStats(partitionTopic0, false); - assertEquals(internalStats0.cursors.keySet(), new TreeSet<>(Lists.newArrayList(Codec.encode(subName)))); + assertEquals(internalStats0.cursors.keySet(), Set.of(Codec.encode(subName))); PersistentTopicInternalStats internalStats1 = admin.topics().getInternalStats(partitionTopic1, false); - assertEquals(internalStats1.cursors.keySet(), new TreeSet<>(Lists.newArrayList(Codec.encode(subName)))); + assertEquals(internalStats1.cursors.keySet(), Set.of(Codec.encode(subName))); // expected internal stats PartitionedTopicMetadata partitionedTopicMetadata = new PartitionedTopicMetadata(2); @@ -1294,7 +1294,7 @@ public void testDeleteTenantForcefully() throws Exception { String namespace = tenant + "/my-ns"; admin.namespaces().createNamespace("my-tenant/my-ns", Set.of("test")); - assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList("my-tenant/my-ns")); + assertEquals(admin.namespaces().getNamespaces(tenant), List.of("my-tenant/my-ns")); // create topic String topic = namespace + "/my-topic"; @@ -1350,7 +1350,7 @@ public void testDeleteNamespaceForcefully() throws Exception { String namespace = tenant + "/my-ns"; admin.namespaces().createNamespace("my-tenant/my-ns", Set.of("test")); - assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList("my-tenant/my-ns")); + assertEquals(admin.namespaces().getNamespaces(tenant), List.of("my-tenant/my-ns")); // create topic String topic = namespace + "/my-topic"; @@ -1399,7 +1399,7 @@ public void testForceDeleteTenantNotAllowed() throws Exception { String namespace = tenant + "/my-ns"; admin.namespaces().createNamespace("my-tenant/my-ns", Set.of("test")); - assertEquals(admin.namespaces().getNamespaces(tenant), Lists.newArrayList("my-tenant/my-ns")); + assertEquals(admin.namespaces().getNamespaces(tenant), List.of("my-tenant/my-ns")); // create topic String topic = namespace + "/my-topic"; @@ -1436,7 +1436,7 @@ public void testNamespaceSplitBundle() throws Exception { .create(); producer.send("message".getBytes()); publishMessagesOnPersistentTopic(topicName, 0); - assertEquals(admin.topics().getList(namespace), Lists.newArrayList(topicName)); + assertEquals(admin.topics().getList(namespace), List.of(topicName)); try { admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true, null); @@ -1458,7 +1458,7 @@ public void testNamespaceSplitBundle() throws Exception { public void testNamespaceSplitBundleWithTopicCountEquallyDivideAlgorithm() throws Exception { // Force to create a topic final String namespace = "prop-xyz/ns1"; - List topicNames = Lists.newArrayList( + List topicNames = List.of( (new StringBuilder("persistent://")).append(namespace).append("/topicCountEquallyDivideAlgorithum-1").toString(), (new StringBuilder("persistent://")).append(namespace).append("/topicCountEquallyDivideAlgorithum-2").toString()); @@ -1931,12 +1931,12 @@ public void testDeleteSubscription() throws Exception { // create a topic and produce some messages publishMessagesOnPersistentTopic(persistentTopicName, 5); assertEquals(admin.topics().getList("prop-xyz/ns1"), - Lists.newArrayList(persistentTopicName)); + List.of(persistentTopicName)); // create the subscription by PulsarAdmin admin.topics().createSubscription(persistentTopicName, subName, MessageId.earliest); - assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList(subName)); + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName)); // create consumer and subscription @Cleanup @@ -1956,7 +1956,7 @@ public void testDeleteSubscription() throws Exception { } // failed to delete the subscription - assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList(subName)); + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName)); // try to delete the subscription with a connected consumer forcefully admin.topics().deleteSubscription(persistentTopicName, subName, true); @@ -2076,9 +2076,9 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub"); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"), - Lists.newArrayList("my-sub-1", "my-sub-2")); + List.of("my-sub-1", "my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds1"), - Lists.newArrayList("my-sub-1")); + List.of("my-sub-1")); consumer2.close(); consumer5.close(); @@ -2086,7 +2086,7 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub-1"); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"), - Lists.newArrayList("my-sub-2")); + List.of("my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds1"), new ArrayList<>()); } @@ -2218,9 +2218,9 @@ public void testJacksonWithTypeDifferences() throws Exception { @Test public void testBackwardCompatibility() throws Exception { - assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz")); + assertEquals(admin.tenants().getTenants(), List.of("prop-xyz")); assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAdminRoles(), - Lists.newArrayList("role1", "role2")); + List.of("role1", "role2")); assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), Set.of("test")); // Try to deserialize property JSON with IncompatibleTenantAdmin format @@ -2266,7 +2266,7 @@ public void persistentTopicsCursorReset(String topicName) throws Exception { .subscriptionType(SubscriptionType.Exclusive) .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0, false); @@ -2319,7 +2319,7 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep .subscriptionType(SubscriptionType.Exclusive) .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0, false); @@ -2475,7 +2475,7 @@ public void partitionedTopicsCursorReset(String topicName) throws Exception { List topics = admin.topics().getList("prop-xyz/ns1"); assertEquals(topics.size(), 4); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0, false); Thread.sleep(1); @@ -2517,7 +2517,7 @@ public void persistentTopicsInvalidCursorReset() throws Exception { String topicName = "persistent://prop-xyz/ns1/invalidcursorreset"; // Force to create a topic publishMessagesOnPersistentTopic(topicName, 0); - assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList(topicName)); + assertEquals(admin.topics().getList("prop-xyz/ns1"), List.of(topicName)); // create consumer and subscription @Cleanup @@ -2528,7 +2528,7 @@ public void persistentTopicsInvalidCursorReset() throws Exception { Consumer consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 10); @@ -2601,7 +2601,7 @@ public void testPersistentTopicsExpireMessages() throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 0); assertEquals(admin.topics().getList("prop-xyz/ns1"), - Lists.newArrayList("persistent://prop-xyz/ns1/ds2")); + List.of("persistent://prop-xyz/ns1/ds2")); // create consumer and subscription @Cleanup @@ -2685,7 +2685,7 @@ public void testPersistentTopicsExpireMessagesInvalidPartitionIndex() throws Exc // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2-partition-2", 0); assertEquals(admin.topics().getList("prop-xyz/ns1"), - Lists.newArrayList("persistent://prop-xyz/ns1/ds2-partition-2")); + List.of("persistent://prop-xyz/ns1/ds2-partition-2")); // create consumer and subscription @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index d035d7c4290e9..c1bfde505bf61 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Response; @@ -141,7 +140,7 @@ public void setup() throws Exception { namespaces.setServletContext(new MockServletContext()); namespaces.setPulsar(pulsar); doReturn("test").when(namespaces).clientAppId(); - doReturn(new TreeSet<>(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters(); + doReturn(Set.of("use", "usw", "usc", "global")).when(namespaces).clusters(); doNothing().when(namespaces).validateAdminAccessForTenant("my-tenant"); doNothing().when(namespaces).validateAdminAccessForTenant("other-tenant"); doNothing().when(namespaces).validateAdminAccessForTenant("new-property"); @@ -159,7 +158,7 @@ public void setup() throws Exception { persistentTopics.setPulsar(pulsar); doReturn("test").when(persistentTopics).clientAppId(); doReturn("persistent").when(persistentTopics).domain(); - doReturn(new TreeSet<>(Lists.newArrayList("use", "usw", "usc"))).when(persistentTopics).clusters(); + doReturn(Set.of("use", "usw", "usc")).when(persistentTopics).clusters(); doNothing().when(persistentTopics).validateAdminAccessForTenant("my-tenant"); doNothing().when(persistentTopics).validateAdminAccessForTenant("other-tenant"); doNothing().when(persistentTopics).validateAdminAccessForTenant("prop-xyz"); @@ -481,7 +480,7 @@ public void properties() throws Throwable { verify(properties, times(2)).validateSuperUserAccessAsync(); response = asyncRequests(ctx -> properties.getTenants(ctx)); - assertEquals(response, Lists.newArrayList("test-property")); + assertEquals(response, List.of("test-property")); verify(properties, times(3)).validateSuperUserAccessAsync(); response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "test-property")); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java index fdf6fc562db68..ab568ac5eeccd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java @@ -18,7 +18,14 @@ */ package org.apache.pulsar.broker.admin; -import com.google.common.collect.Lists; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -33,16 +40,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertThrows; - @Test(groups = "broker-admin") public class AnalyzeBacklogSubscriptionTest extends ProducerConsumerBase { @@ -78,7 +75,7 @@ private void simpleAnalyzeBacklogTest(boolean batching) throws Exception { String subName = "sub-1"; admin.topics().createSubscription(topic, subName, MessageId.latest); - assertEquals(admin.topics().getSubscriptions(topic), Lists.newArrayList("sub-1")); + assertEquals(admin.topics().getSubscriptions(topic), List.of("sub-1")); verifyBacklog(topic, subName, 0, 0); @@ -178,7 +175,7 @@ public void partitionedTopicNotAllowed() throws Exception { String subName = "sub-1"; admin.topics().createPartitionedTopic(topic, 2); admin.topics().createSubscription(topic, subName, MessageId.latest); - assertEquals(admin.topics().getSubscriptions(topic), Lists.newArrayList("sub-1")); + assertEquals(admin.topics().getSubscriptions(topic), List.of("sub-1")); // you cannot use this feature on a partitioned topic assertThrows(PulsarAdminException.NotAllowedException.class, () -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java index ada26ee6c56c9..b5d5ffc046705 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java @@ -22,26 +22,24 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - -import com.google.common.collect.Lists; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.io.IOException; import javax.ws.rs.ClientErrorException; import javax.ws.rs.core.Response.Status; - import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; @@ -49,8 +47,8 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -95,7 +93,7 @@ public void createSubscriptionSingleTopic() throws Exception { Status.CONFLICT.getStatusCode()); } - assertEquals(admin.topics().getSubscriptions(topic), Lists.newArrayList("sub-1")); + assertEquals(admin.topics().getSubscriptions(topic), List.of("sub-1")); Producer p1 = pulsarClient.newProducer().topic(topic).create(); p1.send("test-1".getBytes()); @@ -131,7 +129,7 @@ public void createSubscriptionOnPartitionedTopic() throws Exception { for (int i = 0; i < 10; i++) { assertEquals(admin.topics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()), - Lists.newArrayList("sub-1")); + List.of("sub-1")); } } @@ -157,7 +155,7 @@ public void createSubscriptionOnPartitionedTopicWithPartialFailure() throws Exce for (int i = 0; i < 10; i++) { assertEquals( admin.topics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()), - Lists.newArrayList("sub-1")); + List.of("sub-1")); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java index 15bb586607799..1758e13d735b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java @@ -19,8 +19,8 @@ package org.apache.pulsar.broker.admin; import static org.testng.Assert.assertEquals; -import com.google.common.collect.Lists; import java.util.Collections; +import java.util.List; import java.util.Set; import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; @@ -100,7 +100,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 20); assertEquals(admin.topics().getSubscriptions( - TopicName.get(partitionedTopicName).getPartition(15).toString()), Lists.newArrayList("sub-1")); + TopicName.get(partitionedTopicName).getPartition(15).toString()), List.of("sub-1")); consumer.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MaxUnackedMessagesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MaxUnackedMessagesTest.java index 87e4f703bbfa5..921aab3de0700 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MaxUnackedMessagesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MaxUnackedMessagesTest.java @@ -47,7 +47,6 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import org.testng.collections.Lists; @Test(groups = "broker-admin") public class MaxUnackedMessagesTest extends ProducerConsumerBase { @@ -103,7 +102,7 @@ public void testMaxUnackedMessagesOnSubscription() throws Exception { Consumer consumer1 = consumerBuilder.subscribe(); Consumer consumer2 = consumerBuilder.subscribe(); Consumer consumer3 = consumerBuilder.subscribe(); - List> consumers = Lists.newArrayList(consumer1, consumer2, consumer3); + List> consumers = List.of(consumer1, consumer2, consumer3); waitCacheInit(topicName); admin.topics().setMaxUnackedMessagesOnSubscription(topicName, unackMsgAllowed); Awaitility.await().untilAsserted(() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 6c3fba38642d1..6fd874154fad9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -33,19 +33,18 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import java.lang.reflect.Field; import java.net.URI; import java.net.URL; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -171,7 +170,7 @@ public void setup() throws Exception { doReturn("test").when(namespaces).clientAppId(); doReturn(null).when(namespaces).originalPrincipal(); doReturn(null).when(namespaces).clientAuthData(); - doReturn(new TreeSet<>(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters(); + doReturn(Set.of("use", "usw", "usc", "global")).when(namespaces).clusters(); admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); admin.clusters().createCluster("usw", ClusterData.builder().serviceUrl("http://broker-usw.com:8080").build()); @@ -269,13 +268,16 @@ public void testCreateNamespaces() throws Exception { @Test public void testGetNamespaces() throws Exception { - List expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(), + List expectedList = Arrays.asList(this.testLocalNamespaces.get(0).toString(), this.testLocalNamespaces.get(1).toString()); expectedList.sort(null); assertEquals(namespaces.getNamespacesForCluster(this.testTenant, this.testLocalCluster), expectedList); - expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(), - this.testLocalNamespaces.get(1).toString(), this.testLocalNamespaces.get(2).toString(), - this.testGlobalNamespaces.get(0).toString()); + expectedList = Arrays.asList( + this.testLocalNamespaces.get(0).toString(), + this.testLocalNamespaces.get(1).toString(), + this.testLocalNamespaces.get(2).toString(), + this.testGlobalNamespaces.get(0).toString() + ); expectedList.sort(null); AsyncResponse response = mock(AsyncResponse.class); namespaces.getTenantNamespaces(response, this.testTenant); @@ -510,18 +512,18 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testGlobalNamespaces.get(0).getTenant(), this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(), - Lists.newArrayList("use", "usw"))); + List.of("use", "usw"))); repCluster = (Set) asyncRequests(rsp -> namespaces.getNamespaceReplicationClusters(rsp, this.testGlobalNamespaces.get(0).getTenant(), this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName())); - assertEquals(repCluster, Lists.newArrayList("use", "usw")); + assertEquals(repCluster, List.of("use", "usw")); try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testGlobalNamespaces.get(0).getTenant(), this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(), - Lists.newArrayList("use", "invalid-cluster"))); + List.of("use", "invalid-cluster"))); fail("should have failed"); } catch (RestException e) { assertEquals(e.getResponse().getStatus(), Status.FORBIDDEN.getStatusCode()); @@ -531,7 +533,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testGlobalNamespaces.get(0).getTenant(), this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(), - Lists.newArrayList("use", "global"))); + List.of("use", "global"))); fail("should have failed"); } catch (RestException e) { // Ok, global should not be allowed in the list of replication clusters @@ -541,7 +543,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, "global", this.testGlobalNamespaces.get(0).getLocalName(), - Lists.newArrayList("use", "invalid-cluster"))); + List.of("use", "invalid-cluster"))); fail("should have failed"); } catch (RestException e) { // Ok, invalid-cluster is an invalid cluster id @@ -553,7 +555,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, "global", - this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use", "usw"))); + this.testGlobalNamespaces.get(0).getLocalName(), List.of("use", "usw"))); fail("should have failed"); } catch (RestException e) { // Ok, usw was not configured in the list of allowed clusters @@ -566,7 +568,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, "global", - this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"))); + this.testGlobalNamespaces.get(0).getLocalName(), List.of("use"))); fail("should have failed"); } catch (RestException e) { assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); @@ -588,7 +590,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { store.invalidateAll(); try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, "global", - this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"))); + this.testGlobalNamespaces.get(0).getLocalName(), List.of("use"))); fail("should have failed"); } catch (RestException e) { assertEquals(e.getResponse().getStatus(), 500); @@ -604,7 +606,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, - "global", "non-existing-ns", Lists.newArrayList("use"))); + "global", "non-existing-ns", List.of("use"))); fail("should have failed"); } catch (RestException e) { assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode()); @@ -637,7 +639,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { try { asyncRequests(rsp -> namespaces.setNamespaceReplicationClusters(rsp, this.testTenant, this.testLocalCluster, - this.testLocalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"))); + this.testLocalNamespaces.get(0).getLocalName(), List.of("use"))); fail("should have failed"); } catch (RestException e) { assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); @@ -647,7 +649,7 @@ public void testGlobalNamespaceReplicationConfiguration() throws Exception { @Test public void testGetBundles() throws Exception { - List boundaries = Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0x80000000", "0xffffffff"); BundlesData bundle = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -783,8 +785,10 @@ public void testDeleteNamespaces() throws Exception { responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); assertEquals(responseCaptor.getValue().getStatus(), Status.NO_CONTENT.getStatusCode()); - List nsList = Lists.newArrayList(this.testLocalNamespaces.get(1).toString(), - this.testLocalNamespaces.get(2).toString()); + List nsList = Arrays.asList( + this.testLocalNamespaces.get(1).toString(), + this.testLocalNamespaces.get(2).toString() + ); nsList.sort(null); assertEquals(asyncRequests(ctx -> namespaces.getTenantNamespaces(ctx, this.testTenant)), nsList); @@ -803,7 +807,7 @@ public void testDeleteNamespaces() throws Exception { public void testDeleteNamespaceWithBundles() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-delete-namespace-with-bundles"; - List boundaries = Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0x80000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -901,7 +905,7 @@ public void testUnloadNamespaces() throws Exception { public void testSplitBundles() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-bundled-namespace-1"; - List boundaries = Lists.newArrayList("0x00000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -940,7 +944,7 @@ public void testSplitBundles() throws Exception { public void testSplitBundleWithUnDividedRange() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-bundled-namespace-1"; - List boundaries = Lists.newArrayList("0x00000000", "0x08375b1a", "0x08375b1b", "0xffffffff"); + List boundaries = List.of("0x00000000", "0x08375b1a", "0x08375b1b", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -967,7 +971,7 @@ public void testSplitBundleWithUnDividedRange() throws Exception { public void testUnloadNamespaceWithBundles() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-bundled-namespace-1"; - List boundaries = Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0x80000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -1030,7 +1034,7 @@ public void testRetention() throws Exception { try { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-bundled-namespace-1"; - List boundaries = Lists.newArrayList("0x00000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) @@ -1101,7 +1105,7 @@ public void testPersistenceUnauthorized() throws Exception { public void testValidateTopicOwnership() throws Exception { URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); String bundledNsLocal = "test-bundled-namespace-1"; - List boundaries = Lists.newArrayList("0x00000000", "0xffffffff"); + List boundaries = List.of("0x00000000", "0xffffffff"); BundlesData bundleData = BundlesData.builder() .boundaries(boundaries) .numBundles(boundaries.size() - 1) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java index 556f9f3ac0544..91b0fdbf079a4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java @@ -25,13 +25,11 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.TreeSet; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.apache.pulsar.broker.admin.v2.Namespaces; @@ -97,7 +95,7 @@ public void setup() throws Exception { doReturn("test").when(namespaces).clientAppId(); doReturn(null).when(namespaces).originalPrincipal(); doReturn(null).when(namespaces).clientAuthData(); - doReturn(new TreeSet<>(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters(); + doReturn(Set.of("use", "usw", "usc", "global")).when(namespaces).clusters(); admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); admin.clusters().createCluster("usw", ClusterData.builder().serviceUrl("http://broker-usw.com:8080").build()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 3a9bd21245bf0..5aef3644f2db4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -30,7 +30,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.common.collect.Lists; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; @@ -212,7 +211,7 @@ public void testGetSubscriptions() { response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-0", true); - verify(response, timeout(5000).times(1)).resume(Lists.newArrayList("test")); + verify(response, timeout(5000).times(1)).resume(List.of("test")); // 6) Delete the subscription response = mock(AsyncResponse.class); @@ -240,7 +239,7 @@ public void testGetSubscriptions() { response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-1", true); - verify(response, timeout(5000).times(1)).resume(Lists.newArrayList("test")); + verify(response, timeout(5000).times(1)).resume(List.of("test")); // response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-0", @@ -249,7 +248,7 @@ public void testGetSubscriptions() { // response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName, true); - verify(response, timeout(5000).times(1)).resume(Lists.newArrayList("test")); + verify(response, timeout(5000).times(1)).resume(List.of("test")); // 9) Delete the partitioned topic response = mock(AsyncResponse.class); @@ -603,7 +602,7 @@ public void testCreatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix when(pulsar.getPulsarResources().getTopicResources() .listPersistentTopicsAsync(NamespaceName.get("my-tenant/my-namespace"))) - .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList( + .thenReturn(CompletableFuture.completedFuture(List.of( "persistent://my-tenant/my-namespace/" + nonPartitionTopicName1, "persistent://my-tenant/my-namespace/" + nonPartitionTopicName2 ))); @@ -1188,7 +1187,7 @@ public void testExamineMessageMetadata() throws Exception { producer.newMessage() .keyBytes("partition123".getBytes()) .orderingKey(new byte[]{0}) - .replicationClusters(Lists.newArrayList("a", "b")) + .replicationClusters(List.of("a", "b")) .sequenceId(112233) .value("data") .send(); @@ -1204,7 +1203,7 @@ public void testExamineMessageMetadata() throws Exception { Assert.assertEquals("partition123".getBytes(), message.getKeyBytes()); Assert.assertTrue(message.hasBase64EncodedKey()); //test arrays - Assert.assertEquals(Lists.newArrayList("a", "b"), message.getReplicateTo()); + Assert.assertEquals(List.of("a", "b"), message.getReplicateTo()); //test string Assert.assertEquals(producer.getProducerName(), message.getProducerName()); //test enum diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 0cf5ed79ea743..22670babcc6f6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -88,7 +88,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import org.testng.collections.Lists; @Slf4j @Test(groups = "broker-admin") @@ -3072,22 +3071,22 @@ public void testShadowTopics() throws Exception { //shadow topic must exist Assert.expectThrows(PulsarAdminException.PreconditionFailedException.class, ()-> - admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic1))); + admin.topics().setShadowTopics(sourceTopic, List.of(shadowTopic1))); //shadow topic must be persistent topic Assert.expectThrows(PulsarAdminException.PreconditionFailedException.class, ()-> admin.topics().setShadowTopics(sourceTopic, - Lists.newArrayList("non-persistent://" + myNamespace + "/shadow-test1-" + UUID.randomUUID()))); + List.of("non-persistent://" + myNamespace + "/shadow-test1-" + UUID.randomUUID()))); pulsarClient.newProducer().topic(shadowTopic1).create().close(); pulsarClient.newProducer().topic(shadowTopic2).create().close(); - admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic1)); + admin.topics().setShadowTopics(sourceTopic, List.of(shadowTopic1)); Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.topics().getShadowTopics(sourceTopic), - Lists.newArrayList(shadowTopic1))); - admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic1, shadowTopic2)); + List.of(shadowTopic1))); + admin.topics().setShadowTopics(sourceTopic, List.of(shadowTopic1, shadowTopic2)); Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.topics().getShadowTopics(sourceTopic), - Lists.newArrayList(shadowTopic1, shadowTopic2))); + List.of(shadowTopic1, shadowTopic2))); admin.topics().removeShadowTopics(sourceTopic); Awaitility.await().untilAsserted(() -> assertNull(admin.topics().getShadowTopics(sourceTopic))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java index f339416c2b9d7..7dce82a3691d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java @@ -25,7 +25,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.net.URL; import java.util.HashMap; @@ -33,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -151,7 +149,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build(); Consumer consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName1) .subscriptionType(SubscriptionType.Shared).subscribe(); - assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList(subName1)); + assertEquals(admin.topics().getSubscriptions(partitionedTopicName), List.of(subName1)); Consumer consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2) .subscriptionType(SubscriptionType.Shared).subscribe(); assertEquals(new HashSet<>(admin.topics().getSubscriptions(partitionedTopicName)), @@ -189,7 +187,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { // test cumulative stats for partitioned topic PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList(subName1, subName2))); + assertEquals(topicStats.getSubscriptions().keySet(), Set.of(subName1, subName2)); assertEquals(topicStats.getSubscriptions().get(subName2).getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get(subName2).getMsgBacklog(), totalMessages); assertEquals(topicStats.getPublishers().size(), 1); @@ -242,12 +240,12 @@ public void nonPersistentTopics() throws Exception { publishMessagesOnTopic("non-persistent://prop-xyz/use/ns1/" + topicName, 10, 0); NonPersistentTopicStats topicStats = admin.nonPersistentTopics().getStats(persistentTopicName); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(topicStats.getSubscriptions().keySet(), Set.of("my-sub")); assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1); assertEquals(topicStats.getPublishers().size(), 0); PersistentTopicInternalStats internalStats = admin.nonPersistentTopics().getInternalStats(persistentTopicName); - assertEquals(internalStats.cursors.keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(internalStats.cursors.keySet(), Set.of("my-sub")); consumer.close(); client.close(); @@ -435,7 +433,7 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Shared).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0); @@ -555,21 +553,21 @@ public void testPeerCluster() throws Exception { admin.clusters().createCluster("us-east2", ClusterData.builder().serviceUrl("http://broker.messaging.east2.example.com:8080").build()); - admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west2"))); - assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), Lists.newArrayList("us-west2")); + admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(List.of("us-west2"))); + assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), List.of("us-west2")); assertNull(admin.clusters().getCluster("us-west2").getPeerClusterNames()); // update cluster with duplicate peer-clusters in the list admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet( - Lists.newArrayList("us-west2", "us-east1", "us-west2", "us-east1", "us-west2", "us-east1"))); + List.of("us-west2", "us-east1", "us-west2", "us-east1", "us-west2", "us-east1"))); assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), - Lists.newArrayList("us-west2", "us-east1")); + List.of("us-west2", "us-east1")); admin.clusters().updatePeerClusterNames("us-west1", null); assertNull(admin.clusters().getCluster("us-west1").getPeerClusterNames()); // Check name validation try { admin.clusters().updatePeerClusterNames("us-west1", - Sets.newLinkedHashSet(Lists.newArrayList("invalid-cluster"))); + Sets.newLinkedHashSet(List.of("invalid-cluster"))); fail("should have failed"); } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); @@ -577,7 +575,7 @@ public void testPeerCluster() throws Exception { // Cluster itselft can't be part of peer-list try { - admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west1"))); + admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(List.of("us-west1"))); fail("should have failed"); } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); @@ -615,9 +613,9 @@ public void testReplicationPeerCluster() throws Exception { admin.namespaces().createNamespace(namespace); admin.clusters().updatePeerClusterNames("us-west1", - Sets.newLinkedHashSet(Lists.newArrayList("us-west2", "us-west3"))); + Sets.newLinkedHashSet(List.of("us-west2", "us-west3"))); assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), - Lists.newArrayList("us-west2", "us-west3")); + List.of("us-west2", "us-west3")); // (1) no conflicting peer Set clusterIds = Set.of("us-east1", "us-east2"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java index 9d90933ee9605..c0863cd73337a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java @@ -203,14 +203,14 @@ public void clusters() throws Exception { ClusterData.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build()); // "test" cluster is part of config-default cluster and it's znode gets created when PulsarService creates // failure-domain znode of this default cluster - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("use", "usw")); assertEquals(admin.clusters().getCluster("use"), ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); admin.clusters().updateCluster("usw", ClusterData.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build()); - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("use", "usw")); assertEquals(admin.clusters().getCluster("usw"), ClusterData.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build()); @@ -219,7 +219,7 @@ public void clusters() throws Exception { .serviceUrl("http://new-broker.messaging.usw.example.com:8080") .serviceUrlTls("https://new-broker.messaging.usw.example.com:4443") .build()); - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw")); + assertEquals(admin.clusters().getClusters(), List.of("use", "usw")); assertEquals(admin.clusters().getCluster("usw"), ClusterData.builder() .serviceUrl("http://new-broker.messaging.usw.example.com:8080") @@ -229,7 +229,7 @@ public void clusters() throws Exception { admin.clusters().deleteCluster("usw"); Thread.sleep(300); - assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use")); + assertEquals(admin.clusters().getClusters(), List.of("use")); admin.namespaces().deleteNamespace("prop-xyz/use/ns1"); admin.clusters().deleteCluster("use"); @@ -588,7 +588,7 @@ public void properties() throws PulsarAdminException { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), allowedClusters); admin.tenants().updateTenant("prop-xyz", tenantInfo); - assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz")); + assertEquals(admin.tenants().getTenants(), List.of("prop-xyz")); assertEquals(admin.tenants().getTenantInfo("prop-xyz"), tenantInfo); @@ -635,9 +635,9 @@ public void namespaces() throws Exception { } assertEquals(admin.namespaces().getNamespaces("prop-xyz"), - Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2")); + List.of("prop-xyz/use/ns1", "prop-xyz/use/ns2")); assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), - Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2")); + List.of("prop-xyz/use/ns1", "prop-xyz/use/ns2")); try { admin.namespaces().createNamespace("prop-xyz/usc/ns1"); @@ -693,7 +693,7 @@ public void namespaces() throws Exception { assertTrue(i < 10); admin.namespaces().deleteNamespace("prop-xyz/use/ns1"); - assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2")); + assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), List.of("prop-xyz/use/ns2")); try { admin.namespaces().unload("prop-xyz/use/ns1"); @@ -719,7 +719,7 @@ public void persistentTopics(String topicName) throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0); assertEquals(admin.topics().getList("prop-xyz/use/ns1"), - Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName)); + List.of("persistent://prop-xyz/use/ns1/" + topicName)); // create consumer and subscription @Cleanup @@ -730,18 +730,18 @@ public void persistentTopics(String topicName) throws Exception { Consumer consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of("my-sub")); publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10); TopicStats topicStats = admin.topics().getStats(persistentTopicName); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(List.of("my-sub"))); assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgBacklog(), 10); assertEquals(topicStats.getPublishers().size(), 0); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName, false); - assertEquals(internalStats.cursors.keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(internalStats.cursors.keySet(), new TreeSet<>(List.of("my-sub"))); List> messages = admin.topics().peekMessages(persistentTopicName, "my-sub", 3); assertEquals(messages.size(), 3); @@ -797,7 +797,7 @@ public void partitionedTopics(String topicName) throws Exception { final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName; admin.topics().createPartitionedTopic(partitionedTopicName, 4); assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/use/ns1"), - Lists.newArrayList(partitionedTopicName)); + List.of(partitionedTopicName)); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4); @@ -819,7 +819,7 @@ public void partitionedTopics(String topicName) throws Exception { Consumer consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(partitionedTopicName), List.of("my-sub")); try { admin.topics().deleteSubscription(partitionedTopicName, "my-sub"); @@ -841,7 +841,7 @@ public void partitionedTopics(String topicName) throws Exception { consumer1.close(); admin.topics().deleteSubscription(partitionedTopicName, "my-sub-1"); - assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(partitionedTopicName), List.of("my-sub")); Producer producer = client.newProducer(Schema.BYTES) .topic(partitionedTopicName) @@ -860,7 +860,7 @@ public void partitionedTopics(String topicName) throws Exception { // test cumulative stats for partitioned topic PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false); - assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(Lists.newArrayList("my-sub"))); + assertEquals(topicStats.getSubscriptions().keySet(), new TreeSet<>(List.of("my-sub"))); assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgBacklog(), 10); assertEquals(topicStats.getPublishers().size(), 1); @@ -974,7 +974,7 @@ public void testNamespaceSplitBundle() throws Exception { .create(); producer.send("message".getBytes()); publishMessagesOnPersistentTopic(topicName, 0); - assertEquals(admin.topics().getList(namespace), Lists.newArrayList(topicName)); + assertEquals(admin.topics().getList(namespace), List.of(topicName)); try { admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true, null); @@ -1004,7 +1004,7 @@ public void testNamespaceSplitBundleConcurrent() throws Exception { .create(); producer.send("message".getBytes()); publishMessagesOnPersistentTopic(topicName, 0); - assertEquals(admin.topics().getList(namespace), Lists.newArrayList(topicName)); + assertEquals(admin.topics().getList(namespace), List.of(topicName)); try { admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", false, null); @@ -1113,13 +1113,13 @@ public void testNamespaceUnloadBundle() throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0); assertEquals(admin.topics().getList("prop-xyz/use/ns1"), - Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2")); + List.of("persistent://prop-xyz/use/ns1/ds2")); // create consumer and subscription Consumer consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1/ds2") .subscriptionName("my-sub").subscribe(); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2"), - Lists.newArrayList("my-sub")); + List.of("my-sub")); // Create producer Producer producer = pulsarClient.newProducer(Schema.BYTES) @@ -1174,13 +1174,13 @@ public void testNamespaceBundleUnload(Integer numBundles) throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1-bundles/ds2", 0); assertEquals(admin.topics().getList("prop-xyz/use/ns1-bundles"), - Lists.newArrayList("persistent://prop-xyz/use/ns1-bundles/ds2")); + List.of("persistent://prop-xyz/use/ns1-bundles/ds2")); // create consumer and subscription Consumer consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2") .subscriptionName("my-sub").subscribe(); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), - Lists.newArrayList("my-sub")); + List.of("my-sub")); // Create producer Producer producer = pulsarClient.newProducer(Schema.BYTES) @@ -1326,9 +1326,9 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub"); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), - Lists.newArrayList("my-sub-1", "my-sub-2")); + List.of("my-sub-1", "my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds1"), - Lists.newArrayList("my-sub-1")); + List.of("my-sub-1")); consumer2.close(); consumer5.close(); @@ -1336,7 +1336,7 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub-1"); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), - Lists.newArrayList("my-sub-2")); + List.of("my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds1"), new ArrayList<>()); } @@ -1456,9 +1456,9 @@ public void testJacksonWithTypeDifferencies() throws Exception { @Test public void testBackwardCompatiblity() throws Exception { - assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz")); + assertEquals(admin.tenants().getTenants(), List.of("prop-xyz")); assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAdminRoles(), - Lists.newArrayList("role1", "role2")); + List.of("role1", "role2")); assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), Set.of("use")); // Try to deserialize property JSON with IncompatiblePropertyAdmin format @@ -1490,7 +1490,7 @@ public void persistentTopicsCursorReset(String topicName) throws Exception { .subscriptionType(SubscriptionType.Exclusive) .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0); @@ -1543,7 +1543,7 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep .subscriptionType(SubscriptionType.Exclusive) .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0); @@ -1619,7 +1619,7 @@ public void partitionedTopicsCursorReset(String topicName) throws Exception { List topics = admin.topics().getList("prop-xyz/use/ns1"); assertEquals(topics.size(), 4); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 5, 0); Thread.sleep(1); @@ -1661,7 +1661,7 @@ public void persistentTopicsInvalidCursorReset() throws Exception { String topicName = "persistent://prop-xyz/use/ns1/invalidcursorreset"; // Force to create a topic publishMessagesOnPersistentTopic(topicName, 0); - assertEquals(admin.topics().getList("prop-xyz/use/ns1"), Lists.newArrayList(topicName)); + assertEquals(admin.topics().getList("prop-xyz/use/ns1"), List.of(topicName)); // create consumer and subscription @Cleanup @@ -1672,7 +1672,7 @@ public void persistentTopicsInvalidCursorReset() throws Exception { Consumer consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub") .subscriptionType(SubscriptionType.Exclusive).subscribe(); - assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub")); + assertEquals(admin.topics().getSubscriptions(topicName), List.of("my-sub")); publishMessagesOnPersistentTopic(topicName, 10); @@ -1746,7 +1746,7 @@ public void testPersistentTopicsExpireMessages() throws Exception { // Force to create a topic publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0); assertEquals(admin.topics().getList("prop-xyz/use/ns1"), - Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2")); + List.of("persistent://prop-xyz/use/ns1/ds2")); // create consumer and subscription @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java index 2945049e3f546..d0243ecdef1dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java @@ -19,26 +19,22 @@ package org.apache.pulsar.broker.auth; import static org.testng.Assert.fail; - -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.testng.annotations.AfterClass; - -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - +import com.google.common.collect.Sets; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.ServerSideErrorException; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException.AuthenticationException; import org.apache.pulsar.client.api.PulsarClientException.AuthorizationException; -import org.apache.pulsar.client.api.Producer; -import com.google.common.collect.Sets; - +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; /** * This test doesn't test much in and off itself. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthenticationServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthenticationServiceTest.java index 15feac95380a6..24beb8f426a97 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthenticationServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthenticationServiceTest.java @@ -21,13 +21,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; - +import com.google.common.collect.Sets; import java.io.IOException; import java.util.Set; - import javax.naming.AuthenticationException; import javax.servlet.http.HttpServletRequest; - import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; @@ -35,8 +33,6 @@ import org.testng.Assert; import org.testng.annotations.Test; -import com.google.common.collect.Sets; - public class AuthenticationServiceTest { private static final String s_authentication_success = "authenticated"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 39a91f72dc742..b533f8ce8b221 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Sets; import java.util.EnumSet; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -30,13 +31,12 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import com.google.common.collect.Sets; @Test(groups = "flaky") public class AuthorizationTest extends MockedPulsarServiceBaseTest { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthenticationProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthenticationProvider.java index a15a433f26eca..3ef9b5ad2d63f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthenticationProvider.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthenticationProvider.java @@ -19,13 +19,10 @@ package org.apache.pulsar.broker.auth; import java.io.IOException; - import javax.naming.AuthenticationException; - import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index d3eea7996a187..62fd69b3811d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.auth; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -47,6 +48,8 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.container.TimeoutHandler; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.PulsarMockBookKeeper; @@ -78,12 +81,11 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.MockZooKeeper; import org.apache.zookeeper.data.ACL; +import org.mockito.Mockito; +import org.mockito.internal.util.MockUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.container.AsyncResponse; -import javax.ws.rs.container.TimeoutHandler; - /** * Base class for all tests that need a Pulsar instance without a ZK and BK cluster. */ @@ -232,6 +234,9 @@ protected final void internalCleanup() throws Exception { // an NPE in shutdown, obscuring the real error if (admin != null) { admin.close(); + if (MockUtil.isMock(admin)) { + Mockito.reset(admin); + } admin = null; } if (pulsarClient != null) { @@ -248,6 +253,7 @@ protected final void internalCleanup() throws Exception { resetConfig(); if (mockBookKeeper != null) { mockBookKeeper.reallyShutdown(); + Mockito.reset(mockBookKeeper); mockBookKeeper = null; } if (mockZooKeeperGlobal != null) { @@ -304,6 +310,9 @@ protected void stopBroker() throws Exception { // set shutdown timeout to 0 for forceful shutdown pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0L); pulsar.close(); + if (MockUtil.isMock(pulsar)) { + Mockito.reset(pulsar); + } pulsar = null; // Simulate cleanup of ephemeral nodes //mockZooKeeper.delete("/loadbalance/brokers/localhost:" + pulsar.getConfiguration().getWebServicePort(), -1); @@ -320,12 +329,15 @@ protected void startBroker() throws Exception { if (admin != null) { admin.close(); + if (MockUtil.isMock(admin)) { + Mockito.reset(admin); + } } PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()); customizeNewPulsarAdminBuilder(pulsarAdminBuilder); - admin = spy(pulsarAdminBuilder.build()); + admin = spyWithoutRecordingInvocations(pulsarAdminBuilder.build()); } protected void customizeNewPulsarAdminBuilder(PulsarAdminBuilder pulsarAdminBuilder) { @@ -338,7 +350,7 @@ protected PulsarService startBroker(ServiceConfiguration conf) throws Exception protected PulsarService startBrokerWithoutAuthorization(ServiceConfiguration conf) throws Exception { conf.setBrokerShutdownTimeoutMs(0L); - PulsarService pulsar = spy(newPulsarService(conf)); + PulsarService pulsar = spyWithoutRecordingInvocations(newPulsarService(conf)); setupBrokerMocks(pulsar); beforePulsarStartMocks(pulsar); pulsar.start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index db2db6cc1dbb0..1ff47a4ca5065 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -74,7 +74,7 @@ public void test() throws Exception { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false); + false, 0); assertFalse(tracker.hasMessageAvailable()); @@ -146,7 +146,7 @@ public void testWithTimer() throws Exception { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - false); + false, 0); assertTrue(tasks.isEmpty()); assertTrue(tracker.addMessage(2, 2, 20)); @@ -187,7 +187,7 @@ public void testAddWithinTickTime() { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - false); + false, 0); clockTime.set(0); @@ -209,7 +209,7 @@ public void testAddMessageWithStrictDelay() { @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 100, clock, - true); + true, 0); clockTime.set(10); @@ -236,7 +236,7 @@ public void testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithSt // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 1000, clock, true); + 1000, clock, true, 0); // Set clock time, then run tracker to inherit clock time as the last tick time. clockTime.set(10000); @@ -274,7 +274,7 @@ public void testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStr // a previous tick run. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 100000, clock, true); + 100000, clock, true, 0); clockTime.set(500000); @@ -299,7 +299,7 @@ public void testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict() throws // Use a short tick time to show that the timer task is run based on the deliverAt time in this scenario. @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, - 500, clock, true); + 500, clock, true, 0); clockTime.set(0); @@ -323,9 +323,11 @@ public void testWithFixedDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + final long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -339,13 +341,13 @@ public void testWithFixedDelays() throws Exception { assertEquals(tracker.getNumberOfDelayedMessages(), 5); assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } assertTrue(tracker.shouldPauseAllDeliveries()); - clockTime.set(InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES * 10); + clockTime.set(fixedDelayLookahead * 10); tracker.getScheduledMessages(100); assertFalse(tracker.shouldPauseAllDeliveries()); @@ -367,9 +369,11 @@ public void testWithMixedDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -381,7 +385,7 @@ public void testWithMixedDelays() throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } @@ -401,9 +405,11 @@ public void testWithNoDelays() throws Exception { Clock clock = mock(Clock.class); when(clock.millis()).then(x -> clockTime.get()); + long fixedDelayLookahead = 100; + @Cleanup InMemoryDelayedDeliveryTracker tracker = new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, - true); + true, fixedDelayLookahead); assertFalse(tracker.hasMessageAvailable()); @@ -415,7 +421,7 @@ public void testWithNoDelays() throws Exception { assertFalse(tracker.shouldPauseAllDeliveries()); - for (int i = 6; i <= InMemoryDelayedDeliveryTracker.DETECT_FIXED_DELAY_LOOKAHEAD_MESSAGES; i++) { + for (int i = 6; i <= fixedDelayLookahead; i++) { assertTrue(tracker.addMessage(i, i, i * 10)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index 8ac7a94d2082e..9e17cf85155e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance; -import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -335,7 +335,7 @@ public void testLoadReportParsing() throws Exception { @Test(enabled = true) public void testDoLoadShedding() throws Exception { - SimpleLoadManagerImpl loadManager = spyWithClassAndConstructorArgs(SimpleLoadManagerImpl.class, pulsar1); + SimpleLoadManagerImpl loadManager = spyWithClassAndConstructorArgsRecordingInvocations(SimpleLoadManagerImpl.class, pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); rd.put("memory", new ResourceUsage(1024, 4096)); rd.put("cpu", new ResourceUsage(10, 100)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java index 84b24fc3121e3..e1edb8a9462b4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java @@ -21,17 +21,15 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - -import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; - +import java.util.List; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; -import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; -import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.BrokerData; import org.apache.pulsar.policies.data.loadbalancer.BundleData; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; import org.testng.annotations.Test; @@ -148,7 +146,7 @@ public void testBrokerWithMultipleBundles() { Multimap bundlesToUnload = os.findBundlesForUnloading(loadData, conf); assertFalse(bundlesToUnload.isEmpty()); - assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-10", "bundle-9")); + assertEquals(bundlesToUnload.get("broker-1"), List.of("bundle-10", "bundle-9")); } @Test @@ -187,7 +185,7 @@ public void testFilterRecentlyUnloaded() { Multimap bundlesToUnload = os.findBundlesForUnloading(loadData, conf); assertFalse(bundlesToUnload.isEmpty()); - assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-8", "bundle-7")); + assertEquals(bundlesToUnload.get("broker-1"), List.of("bundle-8", "bundle-7")); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java index 8461f8ce74c43..adc7d2f9c7370 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java @@ -21,16 +21,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; -import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; -import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.BrokerData; import org.apache.pulsar.policies.data.loadbalancer.BundleData; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -161,7 +161,7 @@ public void testBrokerWithMultipleBundles() { Multimap bundlesToUnload = thresholdShedder.findBundlesForUnloading(loadData, conf); assertFalse(bundlesToUnload.isEmpty()); assertEquals(bundlesToUnload.get("broker-1"), - Lists.newArrayList("bundle-10", "bundle-9", "bundle-8")); + List.of("bundle-10", "bundle-9", "bundle-8")); } @Test @@ -209,7 +209,7 @@ public void testFilterRecentlyUnloaded() { Multimap bundlesToUnload = thresholdShedder.findBundlesForUnloading(loadData, conf); assertFalse(bundlesToUnload.isEmpty()); assertEquals(bundlesToUnload.get("broker-1"), - Lists.newArrayList("bundle-8", "bundle-7", "bundle-6", "bundle-5")); + List.of("bundle-8", "bundle-7", "bundle-6", "bundle-5")); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index cb806d3ccfe5d..ca8408c468d9d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -31,11 +31,14 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.google.common.hash.Hashing; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.URI; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -45,7 +48,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; - import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.commons.collections4.CollectionUtils; @@ -64,13 +66,12 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClient; - -import org.apache.pulsar.common.naming.ServiceUnitId; -import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; -import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.LocalPolicies; @@ -82,10 +83,10 @@ import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; -import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.awaitility.Awaitility; import org.mockito.stubbing.Answer; import org.slf4j.Logger; @@ -95,11 +96,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.hash.Hashing; - @Test(groups = "flaky") public class NamespaceServiceTest extends BrokerTestBase { @@ -186,7 +182,7 @@ public void testSplitMapWithRefreshedStatMap() throws Exception { OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); ManagedLedger ledger = mock(ManagedLedger.class); - when(ledger.getCursors()).thenReturn(Lists.newArrayList()); + when(ledger.getCursors()).thenReturn(new ArrayList<>()); doReturn(CompletableFuture.completedFuture(null)).when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class)); Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); @@ -240,7 +236,7 @@ public void testIsServiceUnitDisabled() throws Exception { OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); ManagedLedger ledger = mock(ManagedLedger.class); - when(ledger.getCursors()).thenReturn(Lists.newArrayList()); + when(ledger.getCursors()).thenReturn(new ArrayList<>()); doReturn(CompletableFuture.completedFuture(null)).when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class)); Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); @@ -263,7 +259,7 @@ public void testRemoveOwnershipNamespaceBundle() throws Exception { OwnershipCache ownershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); ManagedLedger ledger = mock(ManagedLedger.class); - when(ledger.getCursors()).thenReturn(Lists.newArrayList()); + when(ledger.getCursors()).thenReturn(new ArrayList<>()); doReturn(CompletableFuture.completedFuture(null)).when(ownershipCache).disableOwnership(any(NamespaceBundle.class)); Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); @@ -386,7 +382,7 @@ public void testCreateLookupResult() throws Exception { final String listenerUrl = "pulsar://localhost:7000"; final String listenerUrlTls = "pulsar://localhost:8000"; final String listener = "listenerName"; - Map advertisedListeners = Maps.newHashMap(); + Map advertisedListeners = new HashMap<>(); advertisedListeners.put(listener, AdvertisedListener.builder().brokerServiceUrl(new URI(listenerUrl)).brokerServiceUrlTls(new URI(listenerUrlTls)).build()); LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker, null, advertisedListeners); URI uri = new URI(candidateBroker); @@ -570,7 +566,7 @@ public void testSplitLargestBundle() throws Exception { NamespaceName nsname = NamespaceName.get(namespace); NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname); - Map topicCount = Maps.newHashMap(); + Map topicCount = new HashMap<>(); int maxTopics = 0; String maxBundle = null; for (int i = 0; i < totalTopics; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java index 6168c61bb20b4..db29d16c9fff3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.namespace; -import static com.google.common.base.Preconditions.checkNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; @@ -34,6 +33,7 @@ import com.google.common.collect.Range; import com.google.common.hash.Hashing; import java.util.EnumSet; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -250,7 +250,7 @@ public void testGetOwnedServiceUnit() throws Exception { assertFalse(cache.getOwnerAsync(testBundle).get().isPresent()); try { - checkNotNull(cache.getOwnedBundle(testBundle)); + Objects.requireNonNull(cache.getOwnedBundle(testBundle)); fail("Should have failed"); } catch (NullPointerException npe) { // OK for not owned namespace @@ -264,7 +264,7 @@ public void testGetOwnedServiceUnit() throws Exception { "https://localhost:4443", false)), Optional.of(-1L), EnumSet.of(CreateOption.Ephemeral)).join(); try { - checkNotNull(cache.getOwnedBundle(testBundle)); + Objects.requireNonNull(cache.getOwnedBundle(testBundle)); fail("Should have failed"); } catch (NullPointerException npe) { // OK for not owned namespace @@ -284,7 +284,7 @@ public void testGetOwnedServiceUnit() throws Exception { assertEquals(data1.getNativeUrlTls(), "pulsar://otherhost:8884"); assertFalse(data1.isDisabled()); try { - checkNotNull(cache.getOwnedBundle(testBundle)); + Objects.requireNonNull(cache.getOwnedBundle(testBundle)); fail("Should have failed"); } catch (NullPointerException npe) { // OK for not owned namespace diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java index e0e3ec9c16a23..86dff398f9774 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java @@ -257,6 +257,14 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep Assert.assertEquals(rgs.getNumResourceGroups(), 0); } + @Test + public void testClose() throws Exception { + ResourceGroupService service = new ResourceGroupService(pulsar, TimeUnit.MILLISECONDS, null, null); + service.close(); + Assert.assertTrue(service.getAggregateLocalUsagePeriodicTask().isCancelled()); + Assert.assertTrue(service.getCalculateQuotaPeriodicTask().isCancelled()); + } + private ResourceGroupService rgs; int numAnonymousQuotaCalculations; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java index cc1558fed2745..cba15b0631006 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java @@ -22,18 +22,22 @@ import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import com.google.common.collect.ImmutableMap; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.plugin.EntryFilter; @@ -60,8 +64,9 @@ public class AbstractBaseDispatcherTest { @BeforeMethod public void setup() throws Exception { this.svcConfig = mock(ServiceConfiguration.class); + when(svcConfig.isDispatchThrottlingForFilteredEntriesEnabled()).thenReturn(true); this.subscriptionMock = mock(PersistentSubscription.class); - this.helper = new AbstractBaseDispatcherTestHelper(this.subscriptionMock, this.svcConfig); + this.helper = new AbstractBaseDispatcherTestHelper(this.subscriptionMock, this.svcConfig, null); } @Test @@ -87,19 +92,26 @@ public void testFilterEntriesForConsumerOfEntryFilter() throws Exception { EntryFilterWithClassLoader mockFilter = mock(EntryFilterWithClassLoader.class); when(mockFilter.filterEntry(any(Entry.class), any(FilterContext.class))).thenReturn( EntryFilter.FilterResult.REJECT); - ImmutableMap entryFilters = ImmutableMap.of("key", mockFilter); + Map entryFilters = Map.of("key", mockFilter); when(mockTopic.getEntryFilters()).thenReturn(entryFilters); + DispatchRateLimiter subscriptionDispatchRateLimiter = mock(DispatchRateLimiter.class); - this.helper = new AbstractBaseDispatcherTestHelper(this.subscriptionMock, this.svcConfig); + this.helper = new AbstractBaseDispatcherTestHelper(this.subscriptionMock, this.svcConfig, + subscriptionDispatchRateLimiter); List entries = new ArrayList<>(); - entries.add(EntryImpl.create(1, 2, createMessage("message1", 1))); + Entry e = EntryImpl.create(1, 2, createMessage("message1", 1)); + long expectedBytePermits = e.getLength(); + entries.add(e); SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); EntryBatchSizes batchSizes = EntryBatchSizes.get(entries.size()); - // - int size = this.helper.filterEntriesForConsumer(entries, batchSizes, sendMessageInfo, null, null, false, null); + + ManagedCursor cursor = mock(ManagedCursor.class); + + int size = this.helper.filterEntriesForConsumer(entries, batchSizes, sendMessageInfo, null, cursor, false, null); assertEquals(size, 0); + verify(subscriptionDispatchRateLimiter).tryDispatchPermit(1, expectedBytePermits); } @Test @@ -201,9 +213,18 @@ private ByteBuf createDelayedMessage(String message, int sequenceId) { private static class AbstractBaseDispatcherTestHelper extends AbstractBaseDispatcher { + private final Optional dispatchRateLimiter; + protected AbstractBaseDispatcherTestHelper(Subscription subscription, - ServiceConfiguration serviceConfig) { + ServiceConfiguration serviceConfig, + DispatchRateLimiter rateLimiter) { super(subscription, serviceConfig); + dispatchRateLimiter = Optional.ofNullable(rateLimiter); + } + + @Override + public Optional getRateLimiter() { + return dispatchRateLimiter; } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index b100c3a977ab2..fccc7abc66854 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -22,11 +22,11 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.beust.jcommander.internal.Maps; import com.google.common.collect.Sets; import java.net.URL; import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -183,7 +183,7 @@ private void rolloverStats() { @Test public void testBacklogQuotaWithReader() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -268,7 +268,7 @@ private TopicStats getTopicStats(String topic1) throws PulsarAdminException { @Test public void testTriggerBacklogQuotaSizeWithReader() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -340,7 +340,7 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { @Test public void testTriggerBacklogTimeQuotaWithReader() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -407,7 +407,7 @@ public void testTriggerBacklogTimeQuotaWithReader() throws Exception { @Test public void testConsumerBacklogEvictionSizeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -442,7 +442,7 @@ public void testConsumerBacklogEvictionSizeQuota() throws Exception { @Test public void testConsumerBacklogEvictionTimeQuotaPrecise() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) @@ -484,7 +484,7 @@ public void testConsumerBacklogEvictionTimeQuotaPrecise() throws Exception { @Test(timeOut = 60000) public void testConsumerBacklogEvictionTimeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) @@ -530,7 +530,7 @@ public void testConsumerBacklogEvictionTimeQuota() throws Exception { @Test public void testConsumerBacklogEvictionTimeQuotaWithEmptyLedger() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) @@ -577,7 +577,7 @@ public void testConsumerBacklogEvictionTimeQuotaWithEmptyLedger() throws Excepti @Test public void testConsumerBacklogEvictionWithAckSizeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -612,7 +612,7 @@ public void testConsumerBacklogEvictionWithAckSizeQuota() throws Exception { @Test public void testConsumerBacklogEvictionWithAckTimeQuotaPrecise() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) @@ -675,7 +675,7 @@ private Producer createProducer(PulsarClient client, String topic) @Test public void testConsumerBacklogEvictionWithAckTimeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build(); @@ -745,7 +745,7 @@ public void testConsumerBacklogEvictionWithAckTimeQuota() throws Exception { @Test public void testConcurrentAckAndEviction() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -816,7 +816,7 @@ public void testConcurrentAckAndEviction() throws Exception { @Test public void testNoEviction() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(10 * 1024) @@ -880,7 +880,7 @@ public void testNoEviction() throws Exception { @Test public void testEvictionMulti() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() .limitSize(15 * 1024) @@ -982,7 +982,7 @@ public void testEvictionMulti() throws Exception { @Test public void testAheadProducerOnHold() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitSize(10 * 1024) @@ -1024,7 +1024,7 @@ public void testAheadProducerOnHold() throws Exception { @Test public void testAheadProducerOnHoldTimeout() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitSize(10 * 1024) @@ -1062,7 +1062,7 @@ public void testAheadProducerOnHoldTimeout() throws Exception { @Test public void testProducerException() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitSize(10 * 1024) @@ -1102,7 +1102,7 @@ public void testProducerException() throws Exception { @Test public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitSize(10 * 1024) @@ -1166,7 +1166,7 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { @Test public void testProducerExceptionAndThenUnblockTimeQuotaPrecise() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) @@ -1232,7 +1232,7 @@ public void testProducerExceptionAndThenUnblockTimeQuotaPrecise() throws Excepti @Test public void testProducerExceptionAndThenUnblockTimeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), - Maps.newHashMap()); + new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java index e9c5032063f11..fe2551287c8c6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java @@ -22,11 +22,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; - -import com.google.common.collect.Lists; - -import java.util.Arrays; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Random; @@ -39,7 +36,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; - import lombok.Cleanup; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -130,7 +126,7 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -179,7 +175,7 @@ public void testSimpleBatchProducerWithFixedBatchBytes(CompressionType compressi .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -223,7 +219,7 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio .create(); Random random = new Random(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { // put a random sleep from 0 to 3 ms Thread.sleep(random.nextInt(4)); @@ -259,7 +255,7 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com .compressionType(compressionType).enableBatching(true).create(); Random random = new Random(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { // put a random sleep from 0 to 3 ms Thread.sleep(random.nextInt(4)); @@ -298,7 +294,7 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType, B .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { if (i == 25) { // send a large message @@ -361,7 +357,7 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType, Bat .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("msg-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -509,7 +505,7 @@ public void testSimpleBatchProducerConsumer1kMessages(BatcherBuilder builder) th .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("msg-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -571,7 +567,7 @@ public void testOutOfOrderAcksForBatchMessage() throws Exception { .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("msg-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -640,7 +636,7 @@ public void testNonBatchCumulativeAckAfterBatchPublish(BatcherBuilder builder) t // create producer to publish non batch messages Producer noBatchProducer = pulsarClient.newProducer().topic(topicName).create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("msg-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -697,7 +693,7 @@ public void testBatchAndNonBatchCumulativeAcks(BatcherBuilder builder) throws Ex .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs / 2; i++) { byte[] message = ("msg-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -760,7 +756,7 @@ public void testConcurrentBatchMessageAck(BatcherBuilder builder) throws Excepti .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -814,7 +810,7 @@ public void testOrderingOfKeyBasedBatchMessageContainer() throws PulsarClientExc .subscriptionName(subscriptionName) .subscriptionType(SubscriptionType.Key_Shared) .subscribe(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); String[] keys = new String[]{"key-1", "key-2", "key-3"}; for (int i = 0; i < 10; i++) { byte[] message = ("my-message-" + i).getBytes(); @@ -912,7 +908,7 @@ public void testRetrieveSequenceIdGenerated(BatcherBuilder builder) throws Excep .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); @@ -944,7 +940,7 @@ public void testRetrieveSequenceIdSpecify(BatcherBuilder builder) throws Excepti .batcherBuilder(builder) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.newMessage().sequenceId(i + 100).value(message).sendAsync()); @@ -1008,7 +1004,7 @@ public void testBatchMessageDispatchingAccordingToPermits() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName).batchingMaxMessages(batchMessages) .batchingMaxPublishDelay(500, TimeUnit.MILLISECONDS).enableBatching(true).create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.newMessage().value(message).sendAsync()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index 731c65dd33d33..c15c93c0e9a52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -18,7 +18,13 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.Lists; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.SneakyThrows; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; @@ -33,13 +39,6 @@ import org.awaitility.Awaitility; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; @Test(groups = "broker") public class BatchMessageWithBatchIndexLevelTest extends BatchMessageTest { @@ -78,7 +77,7 @@ public void testBatchMessageAck() { .enableBatching(true) .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("batch-message-" + i).getBytes(); sendFutureList.add(producer.newMessage().value(message).sendAsync()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index e575154a743e5..62469fdac9bfb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -23,9 +23,10 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -390,7 +391,7 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { long ledgerId = lInfo.getLedgerId(); CompletableFuture> ledgerMetaFuture = ledgerManager.readLedgerMetadata(ledgerId); LedgerMetadata ledgerMetadata = ledgerMetaFuture.get().getValue(); - Set ledgerBookies = Sets.newHashSet(); + Set ledgerBookies = new HashSet<>(); ledgerBookies.addAll(ledgerMetadata.getAllEnsembles().values().iterator().next()); assertEquals(ledgerBookies.size(), isolatedBookies.size()); ledgerBookies.removeAll(isolatedBookies); @@ -736,7 +737,7 @@ public void testDeleteIsolationGroup() throws Exception { setDefaultIsolationGroup(brokerBookkeeperClientIsolationGroups, zkClient, defaultBookies); // primary group empty - setDefaultIsolationGroup(tenantNamespaceIsolationGroupsPrimary, zkClient, Sets.newHashSet()); + setDefaultIsolationGroup(tenantNamespaceIsolationGroupsPrimary, zkClient, new HashSet<>()); setDefaultIsolationGroup(tenantNamespaceIsolationGroupsSecondary, zkClient, isolatedBookies); ServiceConfiguration config = new ServiceConfiguration(); @@ -811,7 +812,7 @@ private void assertAffinityBookies(LedgerManager ledgerManager, List long ledgerId = lInfo.getLedgerId(); CompletableFuture> ledgerMetaFuture = ledgerManager.readLedgerMetadata(ledgerId); LedgerMetadata ledgerMetadata = ledgerMetaFuture.get().getValue(); - Set ledgerBookies = Sets.newHashSet(); + Set ledgerBookies = new HashSet<>(); ledgerBookies.addAll(ledgerMetadata.getAllEnsembles().values().iterator().next()); assertEquals(ledgerBookies.size(), defaultBookies.size()); ledgerBookies.removeAll(defaultBookies); @@ -854,7 +855,7 @@ private void setDefaultIsolationGroup(String brokerBookkeeperClientIsolationGrou bookies = new BookiesRackConfiguration(); } - Map bookieInfoMap = Maps.newHashMap(); + Map bookieInfoMap = new HashMap<>(); for (BookieId bkSocket : bookieAddresses) { BookieInfo info = BookieInfo.builder().rack("use").hostname(bkSocket.toString()).build(); bookieInfoMap.put(bkSocket.toString(), info); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 389e9420c025c..f011ef9b84209 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -29,7 +29,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -579,8 +578,8 @@ public void testBrokerServiceNamespaceStats() throws Exception { final String ns1 = "prop/stats1"; final String ns2 = "prop/stats2"; - List nsList = Lists.newArrayList(ns1, ns2); - List> producerList = Lists.newArrayList(); + List nsList = List.of(ns1, ns2); + List> producerList = new ArrayList<>(); BrokerStats brokerStatsClient = admin.brokerStats(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java index 64cde26847fbe..5b1589a7c9ba6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java @@ -22,8 +22,6 @@ import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - -import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; @@ -259,7 +257,7 @@ public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exceptio .statsInterval(0, TimeUnit.SECONDS) .ioThreads(20).connectionsPerBroker(20).build(); upsertLookupPermits(100); - List> consumers = Collections.synchronizedList(Lists.newArrayList()); + List> consumers = Collections.synchronizedList(new ArrayList<>()); @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(10); final int totalConsumers = 8; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DistributedIdGeneratorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DistributedIdGeneratorTest.java index 75334d60beac3..6aa9a53b33674 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DistributedIdGeneratorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DistributedIdGeneratorTest.java @@ -19,8 +19,8 @@ package org.apache.pulsar.broker.service; import static org.testng.Assert.assertEquals; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -86,7 +86,7 @@ public void concurrent() throws Exception { @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); - List results = Collections.synchronizedList(Lists.newArrayList()); + List results = Collections.synchronizedList(new ArrayList<>()); for (int i = 0; i < Threads; i++) { executor.execute(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java index dde0a586f03f2..ff9cb56b7811c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java @@ -20,16 +20,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - -import com.google.common.collect.Lists; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.pulsar.client.api.Range; import org.apache.pulsar.common.api.proto.IntRange; import org.apache.pulsar.common.api.proto.KeySharedMeta; @@ -228,12 +224,12 @@ public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAs Assert.assertEquals(selector.getRangeConsumer().size(),2); final List> testRanges = new ArrayList<>(); - testRanges.add(Lists.newArrayList( + testRanges.add(List.of( new IntRange().setStart(2).setEnd(2), new IntRange().setStart(3).setEnd(3), new IntRange().setStart(4).setEnd(5)) ); - testRanges.add(Lists.newArrayList( + testRanges.add(List.of( new IntRange().setStart(0).setEnd(0), new IntRange().setStart(1).setEnd(2)) ); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java index 76f09377edcb5..38bb301ae38ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java @@ -23,11 +23,10 @@ import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import lombok.Cleanup; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -81,7 +80,7 @@ public void testMessageExpiryAfterTopicUnload() throws Exception { .enableBatching(false) // this makes the test easier and predictable .create(); - List> sendFutureList = Lists.newArrayList(); + List> sendFutureList = new ArrayList<>(); for (int i = 0; i < numMsgs; i++) { byte[] message = ("my-message-" + i).getBytes(); sendFutureList.add(producer.sendAsync(message)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java index 8381ad599f10f..5f145e258cf75 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java @@ -24,10 +24,10 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.fail; - +import com.google.common.collect.Sets; import java.util.LinkedHashSet; +import java.util.List; import java.util.concurrent.TimeUnit; - import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -43,9 +43,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import org.testng.collections.Lists; - -import com.google.common.collect.Sets; @Test(groups = "broker") public class PeerReplicatorTest extends ReplicatorTestBase { @@ -130,7 +127,7 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception { } // set peer-clusters : r3->r1 - admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r1"))); + admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(List.of("r1"))); Producer producer = client3.newProducer().topic(topic1).create(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topic1).get(); assertNotNull(topic); @@ -145,7 +142,7 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception { producer.close(); // set peer-clusters : r3->r2 - admin2.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r2"))); + admin2.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(List.of("r2"))); producer = client3.newProducer().topic(topic2).create(); topic = (PersistentTopic) pulsar2.getBrokerService().getOrCreateTopic(topic2).get(); assertNotNull(topic); @@ -172,7 +169,7 @@ public void testGetPeerClusters() throws Exception { final String mainClusterName = "r1"; assertNull(admin1.clusters().getPeerClusterNames(mainClusterName)); - LinkedHashSet peerClusters = Sets.newLinkedHashSet(Lists.newArrayList("r2", "r3")); + LinkedHashSet peerClusters = Sets.newLinkedHashSet(List.of("r2", "r3")); admin1.clusters().updatePeerClusterNames(mainClusterName, peerClusters); retryStrategically((test) -> { try { @@ -214,8 +211,8 @@ public void testPeerClusterInReplicationClusterListChange() throws Exception { @Cleanup PulsarClient client3 = PulsarClient.builder().serviceUrl(serviceUrl).statsInterval(0, TimeUnit.SECONDS).build(); // set peer-clusters : r3->r1 - admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r1"))); - admin1.clusters().updatePeerClusterNames("r1", Sets.newLinkedHashSet(Lists.newArrayList("r3"))); + admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(List.of("r1"))); + admin1.clusters().updatePeerClusterNames("r1", Sets.newLinkedHashSet(List.of("r3"))); Producer producer = client3.newProducer().topic(topic1).create(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topic1).get(); assertNotNull(topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java index 9fe83f7b11d25..fa382ef3c5e58 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java @@ -23,17 +23,14 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - -import com.google.common.collect.Lists; import com.google.common.collect.Sets; - +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; - import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -123,8 +120,8 @@ private void verifyConsumerInactive(TestConsumerStateEventListener listener, int private static class ActiveInactiveListenerEvent implements ConsumerEventListener { - private final Set activePtns = Sets.newHashSet(); - private final Set inactivePtns = Sets.newHashSet(); + private final Set activePtns = new HashSet<>(); + private final Set inactivePtns = new HashSet<>(); @Override public synchronized void becameActive(Consumer consumer, int partitionId) { @@ -171,7 +168,7 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception { assertTrue(subRef.getDispatcher().isConsumerConnected()); assertEquals(subRef.getDispatcher().getType(), SubType.Failover); - List> futures = Lists.newArrayListWithCapacity(numMsgs); + List> futures = new ArrayList<>(numMsgs); Producer producer = pulsarClient.newProducer().topic(topicName) .enableBatching(false) .messageRoutingMode(MessageRoutingMode.SinglePartition) @@ -332,7 +329,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception { // equal distribution between both consumers int totalMessages = 0; Message msg = null; - Set receivedPtns = Sets.newHashSet(); + Set receivedPtns = new HashSet<>(); while (true) { msg = consumer1.receive(1, TimeUnit.SECONDS); if (msg == null) { @@ -349,7 +346,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception { Assert.assertEquals(totalMessages, numMsgs / 2); - receivedPtns = Sets.newHashSet(); + receivedPtns = new HashSet<>(); while (true) { msg = consumer2.receive(1, TimeUnit.SECONDS); if (msg == null) { @@ -469,7 +466,7 @@ public void testActiveConsumerFailoverWithDelay() throws Exception { final String topicName = "persistent://prop/use/ns-abc/failover-topic3"; final String subName = "sub1"; final int numMsgs = 100; - List> receivedMessages = Lists.newArrayList(); + List> receivedMessages = new ArrayList<>(); ConsumerBuilder consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) .subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> { @@ -496,7 +493,7 @@ public void testActiveConsumerFailoverWithDelay() throws Exception { PersistentSubscription subRef = topicRef.getSubscription(subName); // enqueue messages - List> futures = Lists.newArrayListWithCapacity(numMsgs); + List> futures = new ArrayList<>(numMsgs); Producer producer = pulsarClient.newProducer().topic(topicName) .enableBatching(false) .messageRoutingMode(MessageRoutingMode.SinglePartition) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java index 9564449ffcc89..6d111a9806ef5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java @@ -23,7 +23,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -31,7 +31,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -48,8 +47,8 @@ import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.policies.data.ConsumerStats; -import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.SubscriptionStats; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.eclipse.jetty.util.BlockingArrayQueue; @@ -59,8 +58,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import com.google.common.collect.Lists; - @Test(groups = "broker") public class PersistentQueueE2ETest extends BrokerTestBase { @@ -110,7 +107,7 @@ public void testSimpleConsumerEvents() throws Exception { assertTrue(subRef.getDispatcher().isConsumerConnected()); assertEquals(subRef.getDispatcher().getType(), SubType.Shared); - List> futures = Lists.newArrayListWithCapacity(numMsgs * 2); + List> futures = new ArrayList<>(numMsgs * 2); Producer producer = pulsarClient.newProducer().topic(topicName) .enableBatching(false) .messageRoutingMode(MessageRoutingMode.SinglePartition) @@ -193,7 +190,7 @@ public void testReplayOnConsumerDisconnect() throws Exception { final String subName = "sub3"; final int numMsgs = 100; - final List messagesProduced = Lists.newArrayListWithCapacity(numMsgs); + final List messagesProduced = new ArrayList<>(numMsgs); final List messagesConsumed = new BlockingArrayQueue<>(numMsgs); Consumer consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) @@ -213,7 +210,7 @@ public void testReplayOnConsumerDisconnect() throws Exception { // do nothing }).subscribe(); - List> futures = Lists.newArrayListWithCapacity(numMsgs * 2); + List> futures = new ArrayList<>(numMsgs * 2); Producer producer = pulsarClient.newProducer().topic(topicName).create(); for (int i = 0; i < numMsgs; i++) { String message = "msg-" + i; @@ -286,7 +283,7 @@ public void testRoundRobinBatchDistribution() throws Exception { } }).subscribe(); - List> futures = Lists.newArrayListWithCapacity(numMsgs); + List> futures = new ArrayList<>(numMsgs); Producer producer = pulsarClient.newProducer().topic(topicName).create(); for (int i = 0; i < numMsgs * 3; i++) { String message = "msg-" + i; @@ -302,8 +299,8 @@ public void testRoundRobinBatchDistribution() throws Exception { * i.e. each consumer will get 130 messages. In the 14th round, the balance is 411 - 130*3 = 21. Two consumers * will get another batch of 10 messages (Total: 140) and the 3rd one will get the last one (Total: 131) */ - assertTrue(CollectionUtils.subtract(Lists.newArrayList(140, 140, 131), - Lists.newArrayList(counter1.get(), counter2.get(), counter3.get())).isEmpty()); + assertTrue(CollectionUtils.subtract(List.of(140, 140, 131), + List.of(counter1.get(), counter2.get(), counter3.get())).isEmpty()); consumer1.close(); consumer2.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index cc987a2ddc0b7..29a8834b4baa6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -24,10 +24,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertFalse; -import com.google.common.collect.Lists; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -108,7 +108,7 @@ public void setup(Method m) throws Exception { doReturn(true).when(nsSvc).isServiceUnitActive(any(TopicName.class)); doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).checkTopicOwnership(any(TopicName.class)); - final List addedEntries = Lists.newArrayList(); + final List addedEntries = new ArrayList<>(); for (int i = 0; i < 100; i++) { Position pos = ledger.addEntry("entry".getBytes()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 1404ce7c7988e..2771808a9fa0b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; @@ -43,8 +44,6 @@ import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; @@ -202,6 +201,8 @@ public void setup() throws Exception { doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory(); doReturn(mock(PulsarClientImpl.class)).when(pulsar).getClient(); + doAnswer(invocationOnMock -> CompletableFuture.completedFuture(null)) + .when(mlFactoryMock).getManagedLedgerPropertiesAsync(any()); doAnswer(invocation -> { DeleteLedgerCallback deleteLedgerCallback = invocation.getArgument(1); deleteLedgerCallback.deleteLedgerComplete(null); @@ -230,7 +231,7 @@ public void setup() throws Exception { doReturn(brokerService).when(pulsar).getBrokerService(); }); // Mock serviceCnx. - serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + serverCnx = spyWithClassAndConstructorArgsRecordingInvocations(ServerCnx.class, pulsar); doReturn(true).when(serverCnx).isActive(); doReturn(true).when(serverCnx).isWritable(); doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress(); @@ -369,7 +370,7 @@ public void setMetadataFromEntryData(ByteBuf entryData) { @Test public void testDispatcherMultiConsumerReadFailed() throws Exception { - PersistentTopic topic = spyWithClassAndConstructorArgs(PersistentTopic.class, successTopicName, ledgerMock, brokerService); + PersistentTopic topic = spyWithClassAndConstructorArgsRecordingInvocations(PersistentTopic.class, successTopicName, ledgerMock, brokerService); ManagedCursor cursor = mock(ManagedCursor.class); when(cursor.getName()).thenReturn("cursor"); Subscription subscription = mock(Subscription.class); @@ -381,7 +382,7 @@ public void testDispatcherMultiConsumerReadFailed() throws Exception { @Test public void testDispatcherSingleConsumerReadFailed() throws Exception { - PersistentTopic topic = spyWithClassAndConstructorArgs(PersistentTopic.class, successTopicName, ledgerMock, brokerService); + PersistentTopic topic = spyWithClassAndConstructorArgsRecordingInvocations(PersistentTopic.class, successTopicName, ledgerMock, brokerService); ManagedCursor cursor = mock(ManagedCursor.class); when(cursor.getName()).thenReturn("cursor"); PersistentDispatcherSingleActiveConsumer dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, @@ -776,7 +777,7 @@ public void testChangeSubscriptionType() throws Exception { consumer.close(); SubType previousSubType = SubType.Exclusive; - for (SubType subType : Lists.newArrayList(SubType.Shared, SubType.Failover, SubType.Key_Shared, + for (SubType subType : List.of(SubType.Shared, SubType.Failover, SubType.Key_Shared, SubType.Exclusive)) { Dispatcher previousDispatcher = sub.getDispatcher(); @@ -1874,7 +1875,7 @@ public void testCompactorSubscription() throws Exception { PositionImpl position = new PositionImpl(1, 1); long ledgerId = 0xc0bfefeL; sub.acknowledgeMessage(Collections.singletonList(position), AckType.Cumulative, - ImmutableMap.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId)); + Map.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId)); verify(compactedTopic, Mockito.times(1)).newCompactedLedger(position, ledgerId); } @@ -1882,7 +1883,7 @@ public void testCompactorSubscription() throws Exception { @Test public void testCompactorSubscriptionUpdatedOnInit() throws Exception { long ledgerId = 0xc0bfefeL; - Map properties = ImmutableMap.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId); + Map properties = Map.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId); PositionImpl position = new PositionImpl(1, 1); doAnswer((invokactionOnMock) -> properties).when(cursorMock).getProperties(); @@ -1934,7 +1935,7 @@ public void testCompactionTriggeredAfterThresholdSecondInvocation() throws Excep doReturn(compactPromise).when(compactor).compact(anyString()); ManagedCursor subCursor = mock(ManagedCursor.class); - doReturn(Lists.newArrayList(subCursor)).when(ledgerMock).getCursors(); + doReturn(List.of(subCursor)).when(ledgerMock).getCursors(); doReturn(Compactor.COMPACTION_SUBSCRIPTION).when(subCursor).getName(); Policies policies = new Policies(); @@ -2105,13 +2106,13 @@ public void testCheckInactiveSubscriptions() throws Exception { .concurrencyLevel(1) .build(); // This subscription is connected by consumer. - PersistentSubscription nonDeletableSubscription1 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "nonDeletableSubscription1", cursorMock, false); + PersistentSubscription nonDeletableSubscription1 = spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, "nonDeletableSubscription1", cursorMock, false); subscriptions.put(nonDeletableSubscription1.getName(), nonDeletableSubscription1); // This subscription is not connected by consumer. - PersistentSubscription deletableSubscription1 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "deletableSubscription1", cursorMock, false); + PersistentSubscription deletableSubscription1 = spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, "deletableSubscription1", cursorMock, false); subscriptions.put(deletableSubscription1.getName(), deletableSubscription1); // This subscription is replicated. - PersistentSubscription nonDeletableSubscription2 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "nonDeletableSubscription2", cursorMock, true); + PersistentSubscription nonDeletableSubscription2 = spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, "nonDeletableSubscription2", cursorMock, true); subscriptions.put(nonDeletableSubscription2.getName(), nonDeletableSubscription2); Field field = topic.getClass().getDeclaredField("subscriptions"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/RackAwareTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/RackAwareTest.java index 25ae0e2f1fd2f..6de41a548cca1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/RackAwareTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/RackAwareTest.java @@ -40,10 +40,9 @@ import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.NetworkTopologyImpl; import org.apache.bookkeeper.test.ServerTester; +import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.policies.data.BookieInfo; -import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping; -import org.assertj.core.util.Lists; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -132,7 +131,7 @@ public void testPlacement() throws Exception { .map(Map::values) .flatMap(bookieId -> bookieId.stream().map(rackInfo -> rackInfo.get("rack"))) .collect(Collectors.toSet()); - assertTrue(racks.containsAll(Lists.newArrayList("rack-1", "rack-2"))); + assertTrue(racks.containsAll(List.of("rack-1", "rack-2"))); }); BookKeeper bkc = this.pulsar.getBookKeeperClient(); @@ -284,7 +283,7 @@ public void testRackUpdate() throws Exception { .flatMap(bookieId -> bookieId.stream().map(rackInfo -> rackInfo.get("rack"))) .collect(Collectors.toSet()); assertEquals(racks.size(), 2); - assertTrue(racks.containsAll(Lists.newArrayList("rack-0", "rack-1"))); + assertTrue(racks.containsAll(List.of("rack-0", "rack-1"))); }); Awaitility.await().untilAsserted(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java index 9c426b2a43587..ea9a9a2b9f2b1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java @@ -28,6 +28,7 @@ import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -174,6 +175,14 @@ public void testReplicationSnapshotStopWhenNoTraffic() throws Exception { // create subscription in r1 createReplicatedSubscription(client1, topicName, subscriptionName, true); + // Validate that no snapshots are created before messages are published + Thread.sleep(2 * config1.getReplicatedSubscriptionsSnapshotFrequencyMillis()); + PersistentTopic t1 = (PersistentTopic) pulsar1.getBrokerService() + .getTopic(topicName, false).get().get(); + ReplicatedSubscriptionsController rsc1 = t1.getReplicatedSubscriptionController().get(); + // no snapshot should have been created before any messages are published + assertTrue(rsc1.getLastCompletedSnapshotId().isEmpty()); + @Cleanup PulsarClient client2 = PulsarClient.builder() .serviceUrl(url2.toString()) @@ -197,9 +206,6 @@ public void testReplicationSnapshotStopWhenNoTraffic() throws Exception { Thread.sleep(2 * config1.getReplicatedSubscriptionsSnapshotFrequencyMillis()); // In R1 - PersistentTopic t1 = (PersistentTopic) pulsar1.getBrokerService() - .getTopic(topicName, false).get().get(); - ReplicatedSubscriptionsController rsc1 = t1.getReplicatedSubscriptionController().get(); Position p1 = t1.getLastPosition(); String snapshot1 = rsc1.getLastCompletedSnapshotId().get(); @@ -541,22 +547,35 @@ public void testReplicatedSubscriptionWhenReplicatorProducerIsClosed() throws Ex .statsInterval(0, TimeUnit.SECONDS) .build(); - // create consumer in r1 - @Cleanup - Consumer consumer1 = client1.newConsumer() - .topic(topicName) - .subscriptionName(subscriptionName) - .replicateSubscriptionState(true) - .subscribe(); + { + // create consumer in r1 + @Cleanup + Consumer consumer = client1.newConsumer() + .topic(topicName) + .subscriptionName(subscriptionName) + .replicateSubscriptionState(true) + .subscribe(); - // waiting to replicate topic/subscription to r1->r2 - Awaitility.await().until(() -> pulsar2.getBrokerService().getTopics().containsKey(topicName)); - final PersistentTopic topic2 = (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); - Awaitility.await().untilAsserted(() -> assertTrue(topic2.getReplicators().get("r1").isConnected())); - Awaitility.await().untilAsserted(() -> assertNotNull(topic2.getSubscription(subscriptionName))); + // send one message to trigger replication + @Cleanup + Producer producer = client1.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("message".getBytes(StandardCharsets.UTF_8)); + + assertEquals(readMessages(consumer, new HashSet<>(), 1, false), 1); + + // waiting to replicate topic/subscription to r1->r2 + Awaitility.await().until(() -> pulsar2.getBrokerService().getTopics().containsKey(topicName)); + final PersistentTopic topic2 = (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + Awaitility.await().untilAsserted(() -> assertTrue(topic2.getReplicators().get("r1").isConnected())); + Awaitility.await().untilAsserted(() -> assertNotNull(topic2.getSubscription(subscriptionName))); + } // unsubscribe replicated subscription in r2 admin2.topics().deleteSubscription(topicName, subscriptionName); + final PersistentTopic topic2 = (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); assertNull(topic2.getSubscription(subscriptionName)); // close replicator producer in r2 @@ -581,6 +600,12 @@ public void testReplicatedSubscriptionWhenReplicatorProducerIsClosed() throws Ex // consume 6 messages in r1 Set receivedMessages = new LinkedHashSet<>(); + @Cleanup + Consumer consumer1 = client1.newConsumer() + .topic(topicName) + .subscriptionName(subscriptionName) + .replicateSubscriptionState(true) + .subscribe(); assertEquals(readMessages(consumer1, receivedMessages, numMessages, false), numMessages); // wait for subscription to be replicated diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 9b87b06012e24..09c7e042f58b7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -28,6 +28,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.scurrilous.circe.checksum.Crc32cIntChecksum; import io.netty.buffer.ByteBuf; @@ -51,7 +52,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.Cleanup; - import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -107,7 +107,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import org.testng.collections.Lists; /** * Starts 3 brokers that are in 3 different clusters @@ -148,7 +147,7 @@ public void testConfigChange() throws Exception { // This test is to verify that the config change on global namespace is successfully applied in broker during // runtime. // Run a set of producer tasks to create the topics - List> results = Lists.newArrayList(); + List> results = new ArrayList<>(); for (int i = 0; i < 10; i++) { final TopicName dest = TopicName.get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/topic-" + i)); @@ -526,7 +525,7 @@ public void testReplicationOverrides() throws Exception { assertTrue(consumer3.drained()); // Produce a message not replicated to r2 - producer1.produce(1, producer1.newMessage().replicationClusters(Lists.newArrayList("r1", "r3"))); + producer1.produce(1, producer1.newMessage().replicationClusters(List.of("r1", "r3"))); consumer1.receive(1); assertTrue(consumer2.drained()); consumer3.receive(1); @@ -823,7 +822,7 @@ public void testReplicatorProducerNameWithUserDefinedReplicatorPrefix() throws E @Test(timeOut = 60000, priority = -1) public void testResumptionAfterBacklogRelaxed() throws Exception { - List policies = Lists.newArrayList(); + List policies = new ArrayList<>(); policies.add(RetentionPolicy.producer_exception); policies.add(RetentionPolicy.producer_request_hold); @@ -1098,7 +1097,7 @@ public void testReplicatedCluster() throws Exception { byte[] value = "test".getBytes(); // publish message local only - TypedMessageBuilder msg = producer1.newMessage().replicationClusters(Lists.newArrayList("r1")).value(value); + TypedMessageBuilder msg = producer1.newMessage().replicationClusters(List.of("r1")).value(value); msg.send(); assertEquals(consumer1.receive().getValue(), value); @@ -1498,7 +1497,7 @@ public void testWhenUpdateReplicationCluster() throws Exception { assertTrue(topic.getReplicators().containsKey("r2")); }); - admin1.topics().setReplicationClusters(dest.toString(), Lists.newArrayList("r1")); + admin1.topics().setReplicationClusters(dest.toString(), List.of("r1")); Awaitility.await().untilAsserted(() -> { Set replicationClusters = admin1.topics().getReplicationClusters(dest.toString(), false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java index 7c28f183c9907..d9bde8d4d5ac0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java @@ -20,14 +20,13 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - +import java.util.List; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import org.testng.collections.Lists; @Test(groups = "broker") public class ReplicatorTlsTest extends ReplicatorTestBase { @@ -50,7 +49,7 @@ public void cleanup() throws Exception { @Test public void testReplicationClient() throws Exception { log.info("--- Starting ReplicatorTlsTest::testReplicationClient ---"); - for (BrokerService ns : Lists.newArrayList(ns1, ns2, ns3)) { + for (BrokerService ns : List.of(ns1, ns2, ns3)) { ns.getReplicationClients().forEach((cluster, client) -> { assertTrue(((PulsarClientImpl) client).getConfiguration().isUseTls()); assertEquals(((PulsarClientImpl) client).getConfiguration().getTlsTrustCertsFilePath(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java index 6d108ce675d17..c005b96d004cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxAuthorizationTest.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.service; -import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockBookKeeper; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper; import static org.mockito.ArgumentMatchers.argThat; @@ -41,7 +41,6 @@ import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; -import javax.crypto.SecretKey; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.charset.StandardCharsets; @@ -50,6 +49,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; +import javax.crypto.SecretKey; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.pulsar.broker.PulsarService; @@ -113,7 +113,7 @@ public void beforeMethod() throws Exception { + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); svcConfig.setProperties(properties); - pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); + pulsar = spyWithClassAndConstructorArgsRecordingInvocations(PulsarService.class, svcConfig); doReturn(new DefaultSchemaRegistryService()).when(pulsar).getSchemaRegistryService(); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -135,21 +135,21 @@ public void beforeMethod() throws Exception { doReturn(store).when(pulsar).getLocalMetadataStore(); doReturn(store).when(pulsar).getConfigurationMetadataStore(); - pulsarResources = spyWithClassAndConstructorArgs(PulsarResources.class, store, store); + pulsarResources = spyWithClassAndConstructorArgsRecordingInvocations(PulsarResources.class, store, store); PulsarServiceMockSupport.mockPulsarServiceProps(pulsar, () -> { doReturn(pulsarResources).when(pulsar).getPulsarResources(); }); NamespaceResources namespaceResources = - spyWithClassAndConstructorArgs(NamespaceResources.class, store, store, 30); + spyWithClassAndConstructorArgsRecordingInvocations(NamespaceResources.class, store, store, 30); doReturn(namespaceResources).when(pulsarResources).getNamespaceResources(); - TenantResources tenantResources = spyWithClassAndConstructorArgs(TenantResources.class, store, 30); + TenantResources tenantResources = spyWithClassAndConstructorArgsRecordingInvocations(TenantResources.class, store, 30); doReturn(tenantResources).when(pulsarResources).getTenantResources(); doReturn(CompletableFuture.completedFuture(Optional.of(TenantInfo.builder().build()))).when(tenantResources) .getTenantAsync("public"); - brokerService = spyWithClassAndConstructorArgs(BrokerService.class, pulsar, eventLoopGroup); + brokerService = spyWithClassAndConstructorArgsRecordingInvocations(BrokerService.class, pulsar, eventLoopGroup); BrokerInterceptor interceptor = mock(BrokerInterceptor.class); doReturn(interceptor).when(brokerService).getInterceptor(); PulsarServiceMockSupport.mockPulsarServiceProps(pulsar, () -> { @@ -162,7 +162,7 @@ public void beforeMethod() throws Exception { public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws Exception { doReturn(true).when(svcConfig).isAuthenticateOriginalAuthData(); - ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + ServerCnx serverCnx = spyWithClassAndConstructorArgsRecordingInvocations(ServerCnx.class, pulsar); ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); ChannelPipeline channelPipeline = mock(ChannelPipeline.class); @@ -198,7 +198,7 @@ public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws E assertEquals(serverCnx.getAuthState().getAuthRole(), PROXY_PRINCIPAL); AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + spyWithClassAndConstructorArgsRecordingInvocations(AuthorizationService.class, svcConfig, pulsarResources); doReturn(authorizationService).when(brokerService).getAuthorizationService(); // lookup @@ -268,7 +268,7 @@ public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws E public void testVerifyOriginalPrincipalWithoutAuthDataForwardedFromProxy() throws Exception { doReturn(false).when(svcConfig).isAuthenticateOriginalAuthData(); - ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + ServerCnx serverCnx = spyWithClassAndConstructorArgsRecordingInvocations(ServerCnx.class, pulsar); ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); ChannelPipeline channelPipeline = mock(ChannelPipeline.class); @@ -299,7 +299,7 @@ public void testVerifyOriginalPrincipalWithoutAuthDataForwardedFromProxy() throw assertEquals(serverCnx.getAuthState().getAuthRole(), PROXY_PRINCIPAL); AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + spyWithClassAndConstructorArgsRecordingInvocations(AuthorizationService.class, svcConfig, pulsarResources); doReturn(authorizationService).when(brokerService).getAuthorizationService(); // lookup @@ -360,7 +360,7 @@ public void testVerifyOriginalPrincipalWithoutAuthDataForwardedFromProxy() throw @Test public void testVerifyAuthRoleAndAuthDataFromDirectConnectionBroker() throws Exception { - ServerCnx serverCnx = spyWithClassAndConstructorArgs(ServerCnx.class, pulsar); + ServerCnx serverCnx = spyWithClassAndConstructorArgsRecordingInvocations(ServerCnx.class, pulsar); ChannelHandlerContext channelHandlerContext = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -391,7 +391,7 @@ public void testVerifyAuthRoleAndAuthDataFromDirectConnectionBroker() throws Exc assertEquals(serverCnx.getAuthState().getAuthRole(), CLIENT_PRINCIPAL); AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsarResources); + spyWithClassAndConstructorArgsRecordingInvocations(AuthorizationService.class, svcConfig, pulsarResources); doReturn(authorizationService).when(brokerService).getAuthorizationService(); // lookup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 78e994568e881..afc4d5a7ef766 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -37,7 +37,6 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import com.google.common.collect.Maps; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler; @@ -52,6 +51,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -123,8 +123,8 @@ import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.common.util.GracefulExecutorServicesShutdown; +import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -1422,17 +1422,17 @@ public void testProducerSuccessOnEncryptionRequiredTopic() throws Exception { // Set encryption_required to true Policies policies = mock(Policies.class); policies.encryption_required = true; - policies.topicDispatchRate = Maps.newHashMap(); - policies.clusterSubscribeRate = Maps.newHashMap(); + policies.topicDispatchRate = new HashMap<>(); + policies.clusterSubscribeRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.getPoliciesDispatchRate` - policies.clusterDispatchRate = Maps.newHashMap(); + policies.clusterDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceSubscriptionDispatchRate` - policies.subscriptionDispatchRate = Maps.newHashMap(); + policies.subscriptionDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceReplicatorDispatchRate` - policies.replicatorDispatchRate = Maps.newHashMap(); + policies.replicatorDispatchRate = new HashMap<>(); doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(namespaceResources) .getPoliciesAsync(TopicName.get(encryptionRequiredTopicName).getNamespaceObject()); @@ -1458,17 +1458,17 @@ public void testProducerFailureOnEncryptionRequiredTopic() throws Exception { // Set encryption_required to true Policies policies = mock(Policies.class); policies.encryption_required = true; - policies.topicDispatchRate = Maps.newHashMap(); - policies.clusterSubscribeRate = Maps.newHashMap(); + policies.topicDispatchRate = new HashMap<>(); + policies.clusterSubscribeRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.getPoliciesDispatchRate` - policies.clusterDispatchRate = Maps.newHashMap(); + policies.clusterDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceSubscriptionDispatchRate` - policies.subscriptionDispatchRate = Maps.newHashMap(); + policies.subscriptionDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceReplicatorDispatchRate` - policies.replicatorDispatchRate = Maps.newHashMap(); + policies.replicatorDispatchRate = new HashMap<>(); doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(namespaceResources) .getPoliciesAsync(TopicName.get(encryptionRequiredTopicName).getNamespaceObject()); @@ -1499,16 +1499,16 @@ public void testProducerFailureOnEncryptionRequiredOnBroker() throws Exception { Policies policies = mock(Policies.class); // Namespace policy doesn't require encryption policies.encryption_required = false; - policies.topicDispatchRate = Maps.newHashMap(); - policies.clusterSubscribeRate = Maps.newHashMap(); + policies.topicDispatchRate = new HashMap<>(); + policies.clusterSubscribeRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE - policies.clusterDispatchRate = Maps.newHashMap(); + policies.clusterDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceSubscriptionDispatchRate` - policies.subscriptionDispatchRate = Maps.newHashMap(); + policies.subscriptionDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceReplicatorDispatchRate` - policies.replicatorDispatchRate = Maps.newHashMap(); + policies.replicatorDispatchRate = new HashMap<>(); doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(namespaceResources) .getPoliciesAsync(TopicName.get(encryptionRequiredTopicName).getNamespaceObject()); @@ -1536,17 +1536,17 @@ public void testSendSuccessOnEncryptionRequiredTopic() throws Exception { // Set encryption_required to true Policies policies = mock(Policies.class); policies.encryption_required = true; - policies.topicDispatchRate = Maps.newHashMap(); - policies.clusterSubscribeRate = Maps.newHashMap(); + policies.topicDispatchRate = new HashMap<>(); + policies.clusterSubscribeRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.getPoliciesDispatchRate` - policies.clusterDispatchRate = Maps.newHashMap(); + policies.clusterDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceSubscriptionDispatchRate` - policies.subscriptionDispatchRate = Maps.newHashMap(); + policies.subscriptionDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceReplicatorDispatchRate` - policies.replicatorDispatchRate = Maps.newHashMap(); + policies.replicatorDispatchRate = new HashMap<>(); doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(namespaceResources) .getPoliciesAsync(TopicName.get(encryptionRequiredTopicName).getNamespaceObject()); @@ -1580,17 +1580,17 @@ public void testSendFailureOnEncryptionRequiredTopic() throws Exception { // Set encryption_required to true Policies policies = mock(Policies.class); policies.encryption_required = true; - policies.topicDispatchRate = Maps.newHashMap(); - policies.clusterSubscribeRate = Maps.newHashMap(); + policies.topicDispatchRate = new HashMap<>(); + policies.clusterSubscribeRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.getPoliciesDispatchRate` - policies.clusterDispatchRate = Maps.newHashMap(); + policies.clusterDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceSubscriptionDispatchRate` - policies.subscriptionDispatchRate = Maps.newHashMap(); + policies.subscriptionDispatchRate = new HashMap<>(); // add `clusterDispatchRate` otherwise there will be a NPE // `org.apache.pulsar.broker.service.AbstractTopic.updateNamespaceReplicatorDispatchRate` - policies.replicatorDispatchRate = Maps.newHashMap(); + policies.replicatorDispatchRate = new HashMap<>(); doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(namespaceResources) .getPoliciesAsync(TopicName.get(encryptionRequiredTopicName).getNamespaceObject()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index 66e6c7d1ff948..0e73db5291e00 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -24,7 +24,6 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -367,7 +366,7 @@ public void testConcurrentResetCursor() throws Exception { messageIds.add(msgId); } - List exceptions = Lists.newLinkedList(); + List exceptions = new ArrayList<>(); class ResetCursorThread extends Thread { public void run() { try { @@ -378,7 +377,7 @@ public void run() { } } - List resetCursorThreads = Lists.newLinkedList(); + List resetCursorThreads = new ArrayList<>(); for (int i = 0; i < 4; i ++) { ResetCursorThread thread = new ResetCursorThread(); resetCursorThreads.add(thread); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 6334eb3a74f34..d33d5341e0b40 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -26,11 +26,12 @@ import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; import static org.testng.AssertJUnit.assertTrue; -import com.google.common.collect.Sets; import java.lang.reflect.Field; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -281,7 +282,7 @@ public void testListenerCleanupByPartition() throws Exception { private void prepareData() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); admin.tenants().createTenant("system-topic", - new TenantInfoImpl(Sets.newHashSet(), Sets.newHashSet("test"))); + new TenantInfoImpl(new HashSet<>(), Set.of("test"))); admin.namespaces().createNamespace(NAMESPACE1); admin.namespaces().createNamespace(NAMESPACE2); admin.namespaces().createNamespace(NAMESPACE3); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java index 3cd6fc23de744..10b1e4961ccfb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java @@ -24,8 +24,6 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - -import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.util.Set; import java.util.TreeSet; @@ -165,18 +163,18 @@ public void testContainsStickyKeyHashes(boolean allowOutOfOrderDelivery) throws controller.add(2, 1, 104); if (allowOutOfOrderDelivery) { - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet(100))); - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet(101, 102, 103))); - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet(104, 105))); + assertFalse(controller.containsStickyKeyHashes(Set.of(100))); + assertFalse(controller.containsStickyKeyHashes(Set.of(101, 102, 103))); + assertFalse(controller.containsStickyKeyHashes(Set.of(104, 105))); } else { - assertTrue(controller.containsStickyKeyHashes(Sets.newHashSet(100))); - assertTrue(controller.containsStickyKeyHashes(Sets.newHashSet(101, 102, 103))); - assertTrue(controller.containsStickyKeyHashes(Sets.newHashSet(104, 105))); + assertTrue(controller.containsStickyKeyHashes(Set.of(100))); + assertTrue(controller.containsStickyKeyHashes(Set.of(101, 102, 103))); + assertTrue(controller.containsStickyKeyHashes(Set.of(104, 105))); } - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet())); - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet(99))); - assertFalse(controller.containsStickyKeyHashes(Sets.newHashSet(105, 106))); + assertFalse(controller.containsStickyKeyHashes(Set.of())); + assertFalse(controller.containsStickyKeyHashes(Set.of(99))); + assertFalse(controller.containsStickyKeyHashes(Set.of(105, 106))); } @Test(dataProvider = "allowOutOfOrderDelivery", timeOut = 10000) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 70bdf5e2cd100..72e6caf070b1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -36,7 +36,6 @@ import java.lang.reflect.Field; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -222,50 +221,6 @@ public void teardown() throws Exception { store.close(); } - @Test - public void testCanAcknowledgeAndCommitForTransaction() throws ExecutionException, InterruptedException { - doAnswer((invocationOnMock) -> { - ((AsyncCallbacks.DeleteCallback) invocationOnMock.getArguments()[1]) - .deleteComplete(invocationOnMock.getArguments()[2]); - return null; - }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any()); - - List> positionsPair = new ArrayList<>(); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 1), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 3), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 5), 0)); - - doAnswer((invocationOnMock) -> { - assertTrue(Arrays.deepEquals(((List)invocationOnMock.getArguments()[0]).toArray(), - positionsPair.toArray())); - ((AsyncCallbacks.MarkDeleteCallback) invocationOnMock.getArguments()[2]) - .markDeleteComplete(invocationOnMock.getArguments()[3]); - return null; - }).when(cursorMock).asyncMarkDelete(any(), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); - - // Single ack for txn - persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair); - - // Commit txn - persistentSubscription.endTxn(txnID1.getMostSigBits(), txnID1.getLeastSigBits(), TxnAction.COMMIT_VALUE, -1).get(); - - List positions = new ArrayList<>(); - positions.add(new PositionImpl(3, 100)); - - // Cumulative ack for txn - persistentSubscription.transactionCumulativeAcknowledge(txnID1, positions); - - doAnswer((invocationOnMock) -> { - assertEquals(((PositionImpl) invocationOnMock.getArguments()[0]).compareTo(new PositionImpl(3, 100)), 0); - ((AsyncCallbacks.MarkDeleteCallback) invocationOnMock.getArguments()[2]) - .markDeleteComplete(invocationOnMock.getArguments()[3]); - return null; - }).when(cursorMock).asyncMarkDelete(any(), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); - - // Commit txn - persistentSubscription.endTxn(txnID1.getMostSigBits(), txnID1.getLeastSigBits(), TxnAction.COMMIT_VALUE, -1).get(); - } - @Test public void testCanAcknowledgeAndAbortForTransaction() throws Exception { List> positionsPair = new ArrayList<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java new file mode 100644 index 0000000000000..22bfd70cf8883 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.service.persistent; + + +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ShadowManagedLedgerImpl; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +public class ShadowTopicTest extends BrokerTestBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + baseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + internalCleanup(); + } + + @Test() + public void testNonPartitionedShadowTopicSetup() throws Exception { + String sourceTopic = "persistent://prop/ns-abc/source"; + String shadowTopic = "persistent://prop/ns-abc/shadow"; + //1. test shadow topic setting in topic creation. + admin.topics().createNonPartitionedTopic(sourceTopic); + admin.topics().createShadowTopic(shadowTopic, sourceTopic); + PersistentTopic brokerShadowTopic = + (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(shadowTopic).get().get(); + Assert.assertTrue(brokerShadowTopic.getManagedLedger() instanceof ShadowManagedLedgerImpl); + Assert.assertEquals(brokerShadowTopic.getShadowSourceTopic().get().toString(), sourceTopic); + Assert.assertEquals(admin.topics().getShadowSource(shadowTopic), sourceTopic); + + //2. test shadow topic could be properly loaded after unload. + admin.namespaces().unload("prop/ns-abc"); + Assert.assertTrue(pulsar.getBrokerService().getTopicReference(shadowTopic).isEmpty()); + Assert.assertEquals(admin.topics().getShadowSource(shadowTopic), sourceTopic); + brokerShadowTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(shadowTopic).get().get(); + Assert.assertTrue(brokerShadowTopic.getManagedLedger() instanceof ShadowManagedLedgerImpl); + Assert.assertEquals(brokerShadowTopic.getShadowSourceTopic().get().toString(), sourceTopic); + } + + @Test() + public void testPartitionedShadowTopicSetup() throws Exception { + String sourceTopic = "persistent://prop/ns-abc/source-p"; + String shadowTopic = "persistent://prop/ns-abc/shadow-p"; + String shadowTopicPartition = TopicName.get(shadowTopic).getPartition(0).toString(); + + //1. test shadow topic setting in topic creation. + admin.topics().createPartitionedTopic(sourceTopic, 2); + admin.topics().createShadowTopic(shadowTopic, sourceTopic); + pulsarClient.newProducer().topic(shadowTopic).create().close();//trigger loading partitions. + PersistentTopic brokerShadowTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(shadowTopicPartition).get().get(); + Assert.assertTrue(brokerShadowTopic.getManagedLedger() instanceof ShadowManagedLedgerImpl); + Assert.assertEquals(brokerShadowTopic.getShadowSourceTopic().get().toString(), sourceTopic); + Assert.assertEquals(admin.topics().getShadowSource(shadowTopic), sourceTopic); + + //2. test shadow topic could be properly loaded after unload. + admin.namespaces().unload("prop/ns-abc"); + Assert.assertTrue(pulsar.getBrokerService().getTopicReference(shadowTopic).isEmpty()); + + Assert.assertEquals(admin.topics().getShadowSource(shadowTopic), sourceTopic); + brokerShadowTopic = + (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(shadowTopicPartition).get().get(); + Assert.assertTrue(brokerShadowTopic.getManagedLedger() instanceof ShadowManagedLedgerImpl); + Assert.assertEquals(brokerShadowTopic.getShadowSourceTopic().get().toString(), sourceTopic); + } + + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index a5f8b5ab38f03..bd8017390d807 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.plugin; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.apache.pulsar.client.api.SubscriptionInitialPosition.Earliest; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -27,16 +28,14 @@ import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotNull; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -93,7 +92,7 @@ public void testOverride() throws Exception { EntryFilterWithClassLoader mockFilter = mock(EntryFilterWithClassLoader.class); when(mockFilter.filterEntry(any(Entry.class), any(FilterContext.class))).thenReturn( EntryFilter.FilterResult.REJECT); - ImmutableMap entryFilters = ImmutableMap.of("key", mockFilter); + Map entryFilters = Map.of("key", mockFilter); Field field = topicRef.getClass().getSuperclass().getDeclaredField("entryFilters"); field.setAccessible(true); @@ -102,7 +101,7 @@ public void testOverride() throws Exception { EntryFilterWithClassLoader mockFilter1 = mock(EntryFilterWithClassLoader.class); when(mockFilter1.filterEntry(any(Entry.class), any(FilterContext.class))).thenReturn( EntryFilter.FilterResult.ACCEPT); - ImmutableMap entryFilters1 = ImmutableMap.of("key2", mockFilter1); + Map entryFilters1 = Map.of("key2", mockFilter1); Field field2 = pulsar.getBrokerService().getClass().getDeclaredField("entryFilters"); field2.setAccessible(true); field2.set(pulsar.getBrokerService(), entryFilters1); @@ -165,10 +164,10 @@ public void testFilter() throws Exception { field.setAccessible(true); NarClassLoader narClassLoader = mock(NarClassLoader.class); EntryFilter filter1 = new EntryFilterTest(); - EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); + EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter1, narClassLoader); EntryFilter filter2 = new EntryFilter2Test(); - EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader); - field.set(dispatcher, ImmutableList.of(loader1, loader2)); + EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter2, narClassLoader); + field.set(dispatcher, List.of(loader1, loader2)); Producer producer = pulsarClient.newProducer(Schema.STRING) .enableBatching(false).topic(topic).create(); @@ -263,7 +262,7 @@ public void testFilter() throws Exception { .getTopicReference(topic).get(); Field field1 = topicRef.getClass().getSuperclass().getDeclaredField("entryFilters"); field1.setAccessible(true); - field1.set(topicRef, ImmutableMap.of("1", loader1, "2", loader2)); + field1.set(topicRef, Map.of("1", loader1, "2", loader2)); cleanup(); verify(loader1, times(1)).close(); @@ -293,7 +292,7 @@ public void testFilteredMsgCount() throws Throwable { EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); EntryFilter filter2 = new EntryFilter2Test(); EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader); - field.set(dispatcher, ImmutableList.of(loader1, loader2)); + field.set(dispatcher, List.of(loader1, loader2)); for (int i = 0; i < 10; i++) { producer.send("test"); @@ -368,7 +367,7 @@ public void testEntryFilterRescheduleMessageDependingOnConsumerSharedSubscriptio EntryFilter filter2 = new EntryFilterTest(); EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader); - field.set(dispatcher, ImmutableList.of(loader1, loader2)); + field.set(dispatcher, List.of(loader1, loader2)); for (int i = 0; i < numMessages; i++) { if (i % 2 == 0) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java index 78cb9af48cbf2..a2e3bed7bbf52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.broker.service.schema; -import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; import lombok.Data; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.StringSchema; import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; +import org.apache.pulsar.client.impl.schema.StringSchema; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; @@ -32,12 +33,10 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.Map; - @Test(groups = "broker") public class KeyValueSchemaCompatibilityCheckTest { - private final Map checkers = Maps.newHashMap(); + private final Map checkers = new HashMap<>(); @Data private static class Foo { @@ -63,7 +62,7 @@ protected void setup() { public void testCheckKeyValueAvroCompatibilityFull() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -77,7 +76,7 @@ public void testCheckKeyValueAvroCompatibilityFull() { public void testCheckKeyValueAvroInCompatibilityFull() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -91,7 +90,7 @@ public void testCheckKeyValueAvroInCompatibilityFull() { public void testCheckKeyValueAvroCompatibilityBackward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -105,7 +104,7 @@ public void testCheckKeyValueAvroCompatibilityBackward() { public void testCheckKeyValueAvroInCompatibilityBackward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -119,7 +118,7 @@ public void testCheckKeyValueAvroInCompatibilityBackward() { public void testCheckKeyValueAvroCompatibilityForward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -133,7 +132,7 @@ public void testCheckKeyValueAvroCompatibilityForward() { public void testCheckKeyValueAvroInCompatibilityForward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -147,7 +146,7 @@ public void testCheckKeyValueAvroInCompatibilityForward() { public void testCheckKeyValueJsonCompatibilityFull() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -161,7 +160,7 @@ public void testCheckKeyValueJsonCompatibilityFull() { public void testCheckKeyValueJsonInCompatibilityFull() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -175,7 +174,7 @@ public void testCheckKeyValueJsonInCompatibilityFull() { public void testCheckKeyValueJsonCompatibilityBackward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -189,7 +188,7 @@ public void testCheckKeyValueJsonCompatibilityBackward() { public void testCheckKeyValueJsonInCompatibilityBackWard() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -203,7 +202,7 @@ public void testCheckKeyValueJsonInCompatibilityBackWard() { public void testCheckKeyValueJsonCompatibilityForward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -217,7 +216,7 @@ public void testCheckKeyValueJsonCompatibilityForward() { public void testCheckKeyValueJsonInCompatibilityForward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -231,7 +230,7 @@ public void testCheckKeyValueJsonInCompatibilityForward() { public void testCheckKeyAvroValueJsonCompatibilityFull() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -245,7 +244,7 @@ public void testCheckKeyAvroValueJsonCompatibilityFull() { public void testCheckKeyAvroValueJsonInCompatibilityFull() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -259,7 +258,7 @@ public void testCheckKeyAvroValueJsonInCompatibilityFull() { public void testCheckKeyAvroValueJsonCompatibilityBackward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -273,7 +272,7 @@ public void testCheckKeyAvroValueJsonCompatibilityBackward() { public void testCheckKeyAvroValueJsonInCompatibilityBackward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -287,7 +286,7 @@ public void testCheckKeyAvroValueJsonInCompatibilityBackward() { public void testCheckKeyAvroValueJsonCompatibilityForward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -301,7 +300,7 @@ public void testCheckKeyAvroValueJsonCompatibilityForward() { public void testCheckKeyAvroValueJsonInCompatibilityForward() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); properties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -315,7 +314,7 @@ public void testCheckKeyAvroValueJsonInCompatibilityForward() { public void testCheckKeyJsonValueAvroCompatibilityFull() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -329,7 +328,7 @@ public void testCheckKeyJsonValueAvroCompatibilityFull() { public void testCheckKeyJsonValueAvroInCompatibilityFull() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -343,7 +342,7 @@ public void testCheckKeyJsonValueAvroInCompatibilityFull() { public void testCheckKeyJsonValueAvroCompatibilityBackward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -357,7 +356,7 @@ public void testCheckKeyJsonValueAvroCompatibilityBackward() { public void testCheckKeyJsonValueAvroInCompatibilityBackward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -372,7 +371,7 @@ public void testCheckKeyJsonValueAvroInCompatibilityBackward() { public void testCheckKeyJsonValueAvroCompatibilityForward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -386,7 +385,7 @@ public void testCheckKeyJsonValueAvroCompatibilityForward() { public void testCheckKeyJsonValueAvroInCompatibilityForward() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map properties = Maps.newHashMap(); + Map properties = new HashMap<>(); properties.put("key.schema.type", String.valueOf(SchemaType.JSON)); properties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -400,10 +399,10 @@ public void testCheckKeyJsonValueAvroInCompatibilityForward() { public void testCheckKeyJsonValueAvroKeyTypeInCompatibility() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map fromProperties = Maps.newHashMap(); + Map fromProperties = new HashMap<>(); fromProperties.put("key.schema.type", String.valueOf(SchemaType.JSON)); fromProperties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); - Map toProperties = Maps.newHashMap(); + Map toProperties = new HashMap<>(); toProperties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); toProperties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -417,10 +416,10 @@ public void testCheckKeyJsonValueAvroKeyTypeInCompatibility() { public void testCheckKeyJsonValueAvroValueTypeInCompatibility() { JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map fromProperties = Maps.newHashMap(); + Map fromProperties = new HashMap<>(); fromProperties.put("key.schema.type", String.valueOf(SchemaType.JSON)); fromProperties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); - Map toProperties = Maps.newHashMap(); + Map toProperties = new HashMap<>(); toProperties.put("key.schema.type", String.valueOf(SchemaType.JSON)); toProperties.put("value.schema.type", String.valueOf(SchemaType.JSON)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -434,10 +433,10 @@ public void testCheckKeyJsonValueAvroValueTypeInCompatibility() { public void testCheckPropertiesNullTypeCompatibility() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map fromProperties = Maps.newHashMap(); + Map fromProperties = new HashMap<>(); fromProperties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); fromProperties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); - Map toProperties = Maps.newHashMap(); + Map toProperties = new HashMap<>(); toProperties.put("key.schema.type", String.valueOf(SchemaType.AVRO)); toProperties.put("value.schema.type", String.valueOf(SchemaType.AVRO)); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) @@ -451,8 +450,8 @@ public void testCheckPropertiesNullTypeCompatibility() { public void testCheckSchemaTypeNullCompatibility() { AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Bar.class).build()); - Map fromProperties = Maps.newHashMap(); - Map toProperties = Maps.newHashMap(); + Map fromProperties = new HashMap<>(); + Map toProperties = new HashMap<>(); SchemaData fromSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) .data(KeyValueSchemaImpl.of(fooSchema, barSchema).getSchemaInfo().getSchema()).props(fromProperties).build(); SchemaData toSchemaData = SchemaData.builder().type(SchemaType.KEY_VALUE) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java index 52a684bbf27dc..0a63e7a2b3ed5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java @@ -18,9 +18,28 @@ */ package org.apache.pulsar.broker.service.streamingdispatch; -import com.google.common.base.Charsets; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -40,34 +59,13 @@ import org.mockito.stubbing.Answer; import org.testng.annotations.Test; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; - /** * Tests for {@link StreamingEntryReader} */ @Test(groups = "flaky") public class StreamingEntryReaderTests extends MockedBookKeeperTestCase { - private static final Charset Encoding = Charsets.UTF_8; + private static final Charset Encoding = StandardCharsets.UTF_8; private PersistentTopic mockTopic; private StreamingDispatcher mockDispatcher; private BrokerService mockBrokerService; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java index 801a55ad419be..7eb96e2bd5880 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java @@ -25,7 +25,6 @@ import static org.testng.AssertJUnit.assertEquals; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.io.ByteArrayOutputStream; @@ -409,7 +408,7 @@ public void testAvgMessagesPerEntry() throws Exception { EntryFilterWithClassLoader loader = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter, narClassLoader); - ImmutableMap entryFilters = ImmutableMap.of("filter", loader); + Map entryFilters = Map.of("filter", loader); PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService() .getTopicReference(topic).get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java index b448f00a8ff16..cd07b7f6e25b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.List; @@ -109,7 +110,7 @@ public void testTransactionTopic() throws Exception { managedLedgerConfig.setMaxEntriesPerLedger(2); MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(TransactionCoordinatorID.get(0), pulsar.getManagedLedgerFactory(), managedLedgerConfig, txnLogBufferedWriterConfig, - transactionTimer); + transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar); metrics.generate(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index b1b865727a055..67c60585d2f5b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -20,21 +20,29 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.mock; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.util.Collection; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; -import org.apache.pulsar.broker.service.plugin.*; +import org.apache.pulsar.broker.service.plugin.EntryFilter; +import org.apache.pulsar.broker.service.plugin.EntryFilterTest; +import org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.*; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.policies.data.SubscriptionStats; @@ -187,7 +195,7 @@ public void testSubscriptionStats(final String topic, final String subName, bool EntryFilter filter1 = new EntryFilterTest(); EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); - field.set(dispatcher, ImmutableList.of(loader1)); + field.set(dispatcher, List.of(loader1)); } for (int i = 0; i < 100; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java new file mode 100644 index 0000000000000..ceed05884df2e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionBatchWriterMetricsTest.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.resources.ClusterResources; +import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Test for consuming transaction messages. + */ +@Slf4j +@Test(groups = "broker") +public class TransactionBatchWriterMetricsTest extends MockedPulsarServiceBaseTest { + + private final String clusterName = "test"; + public static final NamespaceName DEFAULT_NAMESPACE = NamespaceName.get("public/default"); + private final String topicNameSuffix = "t-rest-topic"; + private final String topicName = DEFAULT_NAMESPACE.getPersistentTopicName(topicNameSuffix); + + @BeforeClass + public void setup() throws Exception { + super.internalSetup(); + } + + @AfterClass + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + // enable transaction. + conf.setSystemTopicEnabled(true); + conf.setTransactionCoordinatorEnabled(true); + // enabled batch writer. + conf.setTransactionPendingAckBatchedWriteEnabled(true); + conf.setTransactionPendingAckBatchedWriteMaxRecords(10); + conf.setTransactionLogBatchedWriteEnabled(true); + conf.setTransactionLogBatchedWriteMaxRecords(10); + } + + @Override + protected PulsarService startBroker(ServiceConfiguration conf) throws Exception { + PulsarService pulsar = startBrokerWithoutAuthorization(conf); + ensureClusterExists(pulsar, clusterName); + ensureTenantExists(pulsar.getPulsarResources().getTenantResources(), TopicName.PUBLIC_TENANT, clusterName); + ensureNamespaceExists(pulsar.getPulsarResources().getNamespaceResources(), DEFAULT_NAMESPACE, + clusterName); + ensureNamespaceExists(pulsar.getPulsarResources().getNamespaceResources(), NamespaceName.SYSTEM_NAMESPACE, + clusterName); + ensureTopicExists(pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources(), + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, 16); + return pulsar; + } + + @Test + public void testTransactionMetaLogMetrics() throws Exception{ + String metricsLabelCluster = clusterName; + String metricsLabelBroker = pulsar.getAdvertisedAddress().split(":")[0]; + admin.topics().createNonPartitionedTopic(topicName); + + sendAndAckSomeMessages(); + + // call metrics + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(brokerUrl + "/metrics/get"); + Response response = target.request(MediaType.APPLICATION_JSON_TYPE).buildGet().invoke(); + Assert.assertTrue(response.getStatus() / 200 == 1); + List metricsLines = parseResponseEntityToList(response); + + metricsLines = metricsLines.stream().filter(s -> !s.startsWith("#") && s.contains("bufferedwriter")).collect( + Collectors.toList()); + + // verify tc. + String metrics_key_txn_tc_record_count_sum = + "pulsar_txn_tc_bufferedwriter_batch_record_count_sum{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_tc_record_count_sum, metricsLabelCluster, metricsLabelBroker)) + > 0); + String metrics_key_txn_tc_max_delay = + "pulsar_txn_tc_bufferedwriter_flush_trigger_max_delay_total{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_tc_max_delay, metricsLabelCluster, metricsLabelBroker)) + > 0); + String metrics_key_txn_tc_bytes_size = + "pulsar_txn_tc_bufferedwriter_batch_size_bytes_sum{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_tc_bytes_size, metricsLabelCluster, metricsLabelBroker)) + > 0); + // verify pending ack. + String metrics_key_txn_pending_ack_record_count_sum = + "pulsar_txn_pending_ack_store_bufferedwriter_batch_record_count_sum{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_pending_ack_record_count_sum, metricsLabelCluster, metricsLabelBroker)) + > 0); + String metrics_key_txn_pending_ack_max_delay = + "pulsar_txn_pending_ack_store_bufferedwriter_flush_trigger_max_delay_total{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_pending_ack_max_delay, metricsLabelCluster, metricsLabelBroker)) + > 0); + String metrics_key_txn_pending_ack_bytes_size = + "pulsar_txn_pending_ack_store_bufferedwriter_batch_size_bytes_sum{cluster=\"%s\",broker=\"%s\"} "; + Assert.assertTrue(searchMetricsValue(metricsLines, + String.format(metrics_key_txn_pending_ack_bytes_size, metricsLabelCluster, metricsLabelBroker)) + > 0); + + // cleanup. + response.close(); + client.close(); + admin.topics().delete(topicName, true); + } + + private static Double searchMetricsValue(List metricsLines, String key){ + for (int i = 0; i < metricsLines.size(); i++){ + String metricsLine = metricsLines.get(i); + if (metricsLine.startsWith("#")){ + continue; + } + if (metricsLine.startsWith(key)){ + return Double.valueOf(metricsLine.split(" ")[1]); + } + } + return null; + } + + private void sendAndAckSomeMessages() throws Exception { + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topicName) + .sendTimeout(0, TimeUnit.SECONDS) + .enableBatching(false) + .batchingMaxMessages(2) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .subscriptionType(SubscriptionType.Shared) + .topic(topicName) + .isAckReceiptEnabled(true) + .acknowledgmentGroupTime(0, TimeUnit.SECONDS) + .subscriptionName("my-subscription") + .subscribe(); + producer.sendAsync("normal message x".getBytes()).get(); + for (int i = 0; i < 100; i++){ + Transaction transaction = + pulsarClient.newTransaction().withTransactionTimeout(10, TimeUnit.SECONDS).build().get(); + Message msg = consumer.receive(); + producer.newMessage(transaction).value(("tx msg a-" + i).getBytes(StandardCharsets.UTF_8)).sendAsync(); + producer.newMessage(transaction).value(("tx msg b-" + i).getBytes(StandardCharsets.UTF_8)).sendAsync(); + consumer.acknowledgeAsync(msg.getMessageId(), transaction); + transaction.commit().get(); + } + } + + private static void ensureClusterExists(PulsarService pulsar, String cluster) throws Exception { + ClusterResources clusterResources = pulsar.getPulsarResources().getClusterResources(); + ClusterData clusterData = ClusterData.builder() + .serviceUrl(pulsar.getWebServiceAddress()) + .serviceUrlTls(pulsar.getWebServiceAddress()) + .brokerServiceUrl(pulsar.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar.getBrokerServiceUrl()) + .build(); + if (!clusterResources.clusterExists(cluster)) { + clusterResources.createCluster(cluster, clusterData); + } + } + + private static void ensureTopicExists(NamespaceResources.PartitionedTopicResources partitionedTopicResources, + TopicName topicName, int numPartitions) throws Exception { + Optional getResult = + partitionedTopicResources.getPartitionedTopicMetadataAsync(topicName).get(); + if (!getResult.isPresent()) { + partitionedTopicResources.createPartitionedTopic(topicName, new PartitionedTopicMetadata(numPartitions)); + } else { + PartitionedTopicMetadata existsMeta = getResult.get(); + if (existsMeta.partitions < numPartitions) { + partitionedTopicResources.updatePartitionedTopicAsync(topicName, + __ -> new PartitionedTopicMetadata(numPartitions)).get(); + } + } + } + + private static void ensureNamespaceExists(NamespaceResources namespaceResources, NamespaceName namespaceName, + String cluster) throws Exception { + if (!namespaceResources.namespaceExists(namespaceName)) { + Policies policies = new Policies(); + policies.bundles = getBundles(16); + policies.replication_clusters = Collections.singleton(cluster); + namespaceResources.createPolicies(namespaceName, policies); + } else { + namespaceResources.setPolicies(namespaceName, policies -> { + policies.replication_clusters.add(cluster); + return policies; + }); + } + } + + private static void ensureTenantExists(TenantResources tenantResources, String tenant, String cluster) + throws Exception { + if (!tenantResources.tenantExists(tenant)) { + TenantInfoImpl publicTenant = new TenantInfoImpl(Collections.emptySet(), Collections.singleton(cluster)); + tenantResources.createTenant(tenant, publicTenant); + } else { + tenantResources.updateTenantAsync(tenant, ti -> { + ti.getAllowedClusters().add(cluster); + return ti; + }).get(); + } + } + + private List parseResponseEntityToList(Response response) throws Exception { + InputStream inputStream = (InputStream) response.getEntity(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + List list = new ArrayList<>(); + while (true){ + String str = bufferedReader.readLine(); + if (str == null){ + break; + } + list.add(str); + } + return list; + } + + protected PulsarClient newPulsarClient(String url, int intervalInSecs) throws PulsarClientException { + org.apache.pulsar.client.api.ClientBuilder clientBuilder = + PulsarClient.builder() + .serviceUrl(url) + .enableTransaction(true) + .statsInterval(intervalInSecs, TimeUnit.SECONDS); + customizeNewPulsarClientBuilder(clientBuilder); + return createNewPulsarClient(clientBuilder); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index c309f69fd566d..6a078fc19a73d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.common.naming.SystemTopicNames.PENDING_ACK_STORE_SUFFIX; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -691,7 +692,8 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class); doReturn(CompletableFuture.completedFuture( new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, - 500, bufferedWriterConfig, transactionTimer))) + 500, bufferedWriterConfig, transactionTimer, + DISABLED_BUFFERED_WRITER_METRICS))) .when(pendingAckStoreProvider).newPendingAckStore(any()); doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any()); @@ -757,7 +759,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, persistentTopic.getManagedLedger().getConfig(), new TxnLogBufferedWriterConfig(), - transactionTimer); + transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); Class mlTransactionLogClass = MLTransactionLogImpl.class; Field field = mlTransactionLogClass.getDeclaredField("cursor"); field.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java index a277e91f13a10..7421a1b69f9b1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java @@ -20,8 +20,8 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import com.google.common.collect.Lists; import java.lang.reflect.Field; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.PulsarService; @@ -83,7 +83,7 @@ public void testNewTxn() throws TransactionCoordinatorClientException { @Test public void testCommitAndAbort() throws TransactionCoordinatorClientException { TxnID txnID = transactionCoordinatorClient.newTransaction(); - transactionCoordinatorClient.addPublishPartitionToTxn(txnID, Lists.newArrayList("persistent://public/default/testCommitAndAbort")); + transactionCoordinatorClient.addPublishPartitionToTxn(txnID, List.of("persistent://public/default/testCommitAndAbort")); transactionCoordinatorClient.commit(txnID); try { transactionCoordinatorClient.abort(txnID); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java index fe5c7fa196977..0b14f6592fa82 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.fail; @@ -81,7 +82,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { ManagedCursor subCursor = completableFuture.get().openCursor("test"); MLPendingAckStore pendingAckStore = new MLPendingAckStore(completableFuture.get(), cursor, subCursor, 500, - bufferedWriterConfig, transactionTimer); + bufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); Field field = MLPendingAckStore.class.getDeclaredField("managedLedger"); field.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ExceptionHandlerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ExceptionHandlerTest.java index ed708695dc142..35d6ac0ce9b2f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ExceptionHandlerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ExceptionHandlerTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.broker.web; +import static org.eclipse.jetty.http.HttpStatus.INTERNAL_SERVER_ERROR_500; +import static org.eclipse.jetty.http.HttpStatus.PRECONDITION_FAILED_412; +import javax.servlet.http.HttpServletResponse; import lombok.SneakyThrows; import org.apache.pulsar.common.intercept.InterceptException; import org.eclipse.jetty.server.HttpChannel; @@ -25,11 +28,6 @@ import org.mockito.Mockito; import org.testng.annotations.Test; -import javax.servlet.http.HttpServletResponse; - -import static org.eclipse.jetty.http.HttpStatus.INTERNAL_SERVER_ERROR_500; -import static org.eclipse.jetty.http.HttpStatus.PRECONDITION_FAILED_412; - /** * Unit test for ExceptionHandler. */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/PulsarWebResourceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/PulsarWebResourceTest.java index 18e3e9613708c..06f80afda95a4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/PulsarWebResourceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/PulsarWebResourceTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.web; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import javax.servlet.ServletContext; import javax.ws.rs.core.Context; import javax.ws.rs.core.Feature; @@ -36,9 +38,6 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - /** * A base class for testing subclasses of {@link PulsarWebResource}. */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java index c1938adfc5f86..8e6df74d5a0e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/RestExceptionTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.web; import static org.testng.Assert.assertEquals; - import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response.Status; import org.apache.pulsar.common.policies.data.ErrorData; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java index 75b385dba4d5d..088087dce681f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java @@ -22,6 +22,8 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; + +import com.fasterxml.jackson.databind.type.TypeFactory; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -44,6 +46,8 @@ import org.apache.pulsar.PulsarInitialNamespaceSetup; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -86,6 +90,40 @@ public void testReSetupClusterMetadata() throws Exception { assertEquals(data1, data3); } + @DataProvider(name = "bundleNumberForDefaultNamespace") + public static Object[][] bundleNumberForDefaultNamespace() { + return new Object[][] { { 0 }, { 128 } }; + } + + @Test(dataProvider = "bundleNumberForDefaultNamespace") + public void testSetBundleNumberForDefaultNamespace(int bundleNumber) throws Exception { + String[] args = { + "--cluster", "testSetDefaultNamespaceBundleNumber-cluster", + "--zookeeper", "127.0.0.1:" + localZkS.getZookeeperPort(), + "--configuration-store", "127.0.0.1:" + localZkS.getZookeeperPort(), + "--web-service-url", "http://127.0.0.1:8080", + "--web-service-url-tls", "https://127.0.0.1:8443", + "--broker-service-url", "pulsar://127.0.0.1:6650", + "--broker-service-url-tls","pulsar+ssl://127.0.0.1:6651", + "--default-namespace-bundle-number", String.valueOf(bundleNumber) + }; + PulsarClusterMetadataSetup.main(args); + try (ZooKeeper zk = ZooKeeperClient.newBuilder() + .connectString("127.0.0.1:" + localZkS.getZookeeperPort()) + .build()) { + Policies policies = + ObjectMapperFactory.getThreadLocal().readValue( + zk.getData("/admin/policies/public/default", false, null), + TypeFactory.defaultInstance().constructSimpleType(Policies.class, null)); + assertNotNull(policies); + if (bundleNumber > 0) { + assertEquals(policies.bundles.getNumBundles(), bundleNumber); + } else { + assertEquals(policies.bundles.getNumBundles(), 16); + } + } + } + @DataProvider(name = "useMetadataStoreUrl") public static Object[][] useMetadataStoreUrlDataSet() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java index 6b26d1740429a..520c8743cbed2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java @@ -36,6 +36,8 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.mockito.internal.util.MockUtil; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -64,6 +66,12 @@ protected void setup() throws Exception { // Start Broker super.init(); + if (admin != null) { + admin.close(); + if (MockUtil.isMock(admin)) { + Mockito.reset(admin); + } + } admin = getAdmin(ADMIN_TOKEN); admin.clusters().createCluster(configClusterName, ClusterData.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index 0039fc92cb803..a8c3d353e84f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -20,7 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.UUID.randomUUID; -import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.mockito.Mockito.any; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -819,8 +819,10 @@ public void testJsonSchemaProducerConsumerWithSpecifiedReaderAndWriter() throws final String topicName = "persistent://my-property/my-ns/my-topic1"; ObjectMapper mapper = new ObjectMapper(); SchemaReader reader = - spyWithClassAndConstructorArgs(JacksonJsonReader.class, mapper, TestMessageObject.class); - SchemaWriter writer = spyWithClassAndConstructorArgs(JacksonJsonWriter.class, mapper); + spyWithClassAndConstructorArgsRecordingInvocations(JacksonJsonReader.class, mapper, + TestMessageObject.class); + SchemaWriter writer = + spyWithClassAndConstructorArgsRecordingInvocations(JacksonJsonWriter.class, mapper); SchemaDefinition schemaDefinition = new SchemaDefinitionBuilderImpl() .withPojo(TestMessageObject.class) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java index 270d838b61cfb..448f099c386d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java @@ -20,14 +20,12 @@ import static org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse; import static org.apache.pulsar.common.protocol.Commands.newPartitionMetadataResponse; - import com.google.common.collect.Sets; import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -44,20 +42,18 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class LookupRetryTest extends MockedPulsarServiceBaseTest { - private static final Logger log = LoggerFactory.getLogger(LookupRetryTest.class); private static final String subscription = "reader-sub"; private final AtomicInteger connectionsCreated = new AtomicInteger(0); private final ConcurrentHashMap> failureMap = new ConcurrentHashMap<>(); - @BeforeMethod + @BeforeClass @Override protected void setup() throws Exception { conf.setTopicLevelPoliciesEnabled(false); @@ -69,8 +65,6 @@ protected void setup() throws Exception { admin.tenants().createTenant("public", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); admin.namespaces().createNamespace("public/default", Sets.newHashSet("test")); - - connectionsCreated.set(0); } @Override @@ -94,12 +88,18 @@ protected ServerCnx newServerCnx(PulsarService pulsar, String listenerName) thro }; } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); } + @BeforeMethod(alwaysRun = true) + public void reset() { + connectionsCreated.set(0); + failureMap.clear(); + } + PulsarClient newClient() throws Exception { return PulsarClient.builder() .serviceUrl(pulsar.getBrokerServiceUrl()) @@ -244,7 +244,6 @@ public void testCloseConnectionOnBrokerTimeout() throws Exception { try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl) .maxNumberOfRejectedRequestPerConnection(100) - .maxNumberOfRejectedRequestPerConnection(1) .connectionTimeout(2, TimeUnit.SECONDS) .operationTimeout(1, TimeUnit.SECONDS) .lookupTimeout(10, TimeUnit.SECONDS) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java index 769b832ab778d..34f8ff368a960 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java @@ -311,7 +311,7 @@ public void testNegativeAcksWithBatchAckEnabled() throws Exception { .acknowledgmentGroupTime(0, TimeUnit.SECONDS) .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) - .ackTimeout(1000, TimeUnit.MILLISECONDS) + .negativeAckRedeliveryDelay(1, TimeUnit.SECONDS) .subscribe(); @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index aea77bec13678..e3fc05ae0424e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -80,8 +80,8 @@ import org.apache.pulsar.transaction.coordinator.TransactionSubscription; import org.awaitility.Awaitility; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -96,14 +96,21 @@ public class TransactionEndToEndTest extends TransactionTestBase { protected static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; protected static final String TOPIC_MESSAGE_ACK_TEST = NAMESPACE1 + "/message-ack-test"; protected static final int NUM_PARTITIONS = 16; - @BeforeMethod + @BeforeClass protected void setup() throws Exception { conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); admin.topics().createPartitionedTopic(TOPIC_MESSAGE_ACK_TEST, 1); } - @AfterMethod(alwaysRun = true) + protected void resetTopicOutput() throws Exception { + admin.topics().deletePartitionedTopic(TOPIC_OUTPUT, true); + admin.topics().createPartitionedTopic(TOPIC_OUTPUT, TOPIC_PARTITION); + admin.topics().deletePartitionedTopic(TOPIC_MESSAGE_ACK_TEST, true); + admin.topics().createPartitionedTopic(TOPIC_MESSAGE_ACK_TEST, 1); + } + + @AfterClass(alwaysRun = true) protected void cleanup() { super.internalCleanup(); } @@ -167,6 +174,10 @@ private void produceCommitTest(boolean enableBatch) throws Exception { message = consumer.receive(5, TimeUnit.SECONDS); Assert.assertNull(message); + // cleanup. + producer.close(); + consumer.close(); + resetTopicOutput(); log.info("message commit test enableBatch {}", enableBatch); } @@ -175,7 +186,6 @@ public void produceAbortTest() throws Exception { Transaction txn = getTxn(); String subName = "test"; - @Cleanup Producer producer = pulsarClient .newProducer() .topic(TOPIC_OUTPUT) @@ -188,7 +198,6 @@ public void produceAbortTest() throws Exception { producer.newMessage(txn).value(("Hello Txn - " + i).getBytes(UTF_8)).send(); } - @Cleanup Consumer consumer = pulsarClient .newConsumer() .topic(TOPIC_OUTPUT) @@ -253,6 +262,10 @@ public void produceAbortTest() throws Exception { return flag; }); + // cleanup. + producer.close(); + consumer.close(); + resetTopicOutput(); log.info("finished test partitionAbortTest"); } @@ -311,6 +324,11 @@ private void testAckWithTransactionReduceUnAckMessageCount(boolean enableBatch) } } assertTrue(flag); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName, true); } @Test @@ -406,6 +424,11 @@ protected void txnAckTest(boolean batchEnable, int maxBatchSize, Assert.assertTrue(reCommitError.getCause() instanceof TransactionNotFoundException); } } + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(normalTopic, true); } @Test @@ -423,6 +446,11 @@ public void testAfterDeleteTopicOtherTopicCanRecover() throws Exception { String content = "test"; producer.send(content); assertEquals(consumer.receive().getValue(), content); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicTwo, true); } @Test @@ -536,6 +564,10 @@ public void txnMessageAckTest() throws Exception { } assertTrue(exist); + // cleanup. + producer.close(); + consumer.close(); + resetTopicOutput(); log.info("receive transaction messages count: {}", receiveCnt); } @@ -638,6 +670,11 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri message = consumer.receive(1, TimeUnit.SECONDS); Assert.assertNull(message); } + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(normalTopic, true); } private Transaction getTxn() throws Exception { @@ -648,25 +685,6 @@ private Transaction getTxn() throws Exception { .get(); } - private void markDeletePositionCheck(String topic, String subName, boolean equalsWithLastConfirm) throws Exception { - for (int i = 0; i < TOPIC_PARTITION; i++) { - PersistentTopicInternalStats stats = null; - String checkTopic = TopicName.get(topic).getPartition(i).toString(); - for (int j = 0; j < 10; j++) { - stats = admin.topics().getInternalStats(checkTopic, false); - if (stats.lastConfirmedEntry.equals(stats.cursors.get(subName).markDeletePosition)) { - break; - } - Thread.sleep(200); - } - if (equalsWithLastConfirm) { - Assert.assertEquals(stats.cursors.get(subName).markDeletePosition, stats.lastConfirmedEntry); - } else { - Assert.assertNotEquals(stats.cursors.get(subName).markDeletePosition, stats.lastConfirmedEntry); - } - } - } - @Test public void txnMetadataHandlerRecoverTest() throws Exception { String topic = NAMESPACE1 + "/tc-metadata-handler-recover"; @@ -714,6 +732,12 @@ public void txnMetadataHandlerRecoverTest() throws Exception { Message message = consumer.receive(); Assert.assertNotNull(message); } + + // cleanup. + producer.close(); + consumer.close(); + recoverPulsarClient.close(); + admin.topics().delete(topic, true); } @Test @@ -748,9 +772,14 @@ public void produceTxnMessageOrderTest() throws Exception { for (int i = 0; i < 1000; i++) { Message message = consumer.receive(5, TimeUnit.SECONDS); Assert.assertNotNull(message); - Assert.assertEquals(Integer.valueOf(new String(message.getData())), new Integer(i)); + Assert.assertEquals(Integer.valueOf(new String(message.getData())), Integer.valueOf(i)); } } + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @Test @@ -855,10 +884,20 @@ public void produceAndConsumeCloseStateTxnTest() throws Exception { field.setAccessible(true); TransactionImpl.State state = (TransactionImpl.State) field.get(timeoutTxnSkipClientTimeout); assertEquals(state, TransactionImpl.State.ERROR); + + // cleanup. + timeoutTxn.abort(); + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @Test public void testTxnTimeoutAtTransactionMetadataStore() throws Exception{ + Collection transactionMetadataStores = + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores().values(); + long timeoutCountOriginal = transactionMetadataStores.stream() + .mapToLong(store -> store.getMetadataStoreStats().timeoutCount).sum(); TxnID txnID = pulsarServiceList.get(0).getTransactionMetadataStoreService() .newTransaction(new TransactionCoordinatorID(0), 1).get(); Awaitility.await().until(() -> { @@ -869,11 +908,9 @@ public void testTxnTimeoutAtTransactionMetadataStore() throws Exception{ return true; } }); - Collection transactionMetadataStores = - getPulsarServiceList().get(0).getTransactionMetadataStoreService().getStores().values(); long timeoutCount = transactionMetadataStores.stream() .mapToLong(store -> store.getMetadataStoreStats().timeoutCount).sum(); - Assert.assertEquals(timeoutCount, 1); + Assert.assertEquals(timeoutCount, timeoutCountOriginal + 1); } @Test @@ -914,6 +951,11 @@ public void transactionTimeoutTest() throws Exception { assertEquals(reReceiveMessage.getMessageId(), message.getMessageId()); + // cleanup. + consumeTimeoutTxn.abort(); + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @DataProvider(name = "ackType") @@ -979,6 +1021,11 @@ public void txnTransactionRedeliverNullDispatcher(CommandAck.AckType ackType) th } txn.abort().get(); assertTrue(exist); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @Test @@ -1036,6 +1083,13 @@ public void oneTransactionOneTopicWithMultiSubTest() throws Exception { } } assertTrue(flag); + + // cleanup. + txn.abort().get(); + producer.close(); + consumer1.close(); + consumer2.close(); + admin.topics().delete(topic, true); } @Test @@ -1070,6 +1124,11 @@ public void testTxnTimeOutInClient() throws Exception{ Assert.assertTrue(e.getCause() instanceof TransactionCoordinatorClientException .InvalidTxnStatusException); } + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @Test @@ -1138,6 +1197,11 @@ public void testCumulativeAckRedeliverMessages() throws Exception { // then redeliver will not receive any message message = consumer.receive(3, TimeUnit.SECONDS); assertNull(message); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topic, true); } @Test @@ -1172,6 +1236,11 @@ public void testSendTxnMessageTimeout() throws Exception { } catch (PulsarClientException ex) { assertTrue(ex instanceof PulsarClientException.TimeoutException); } + + // cleanup. + transaction.abort().get(); + producer.close(); + admin.topics().delete(topic, true); } @Test @@ -1218,6 +1287,12 @@ public void testAckWithTransactionReduceUnackCountNotInPendingAcks() throws Exce // ack one message, the unack count is 4 assertEquals(getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false) .get().get().getSubscription(subName).getConsumers().get(0).getUnackedMessages(), 4); + + // cleanup. + txn.abort().get(); + consumer.close(); + producer.close(); + admin.topics().delete(topic, true); } @Test @@ -1274,6 +1349,14 @@ public void testSendTxnAckMessageToDLQ() throws Exception { assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 3); assertEquals(value, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); + + // cleanup. + consumer.close(); + deadLetterConsumer.close(); + producer.close(); + admin.topics().delete(String.format("%s-%s" + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX, + topic, subName), true); + admin.topics().delete(topic, true); } @Test @@ -1341,6 +1424,14 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { assertEquals(value1, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); assertEquals(value2, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); + + // cleanup. + consumer.close(); + deadLetterConsumer.close(); + producer.close(); + admin.topics().delete(String.format("%s-%s" + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX, + topic, subName), true); + admin.topics().delete(topic, true); } @Test @@ -1400,5 +1491,11 @@ public void testDelayedTransactionMessages() throws Exception { for (int i = 0; i < 10; i++) { assertTrue(receivedMsgs.contains("msg-" + i)); } + + // cleanup. + sharedConsumer.close(); + failoverConsumer.close(); + producer.close(); + admin.topics().delete(topic, true); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java index 1ef3998c3467d..0b50be807be42 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.SubscriptionType; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; /** @@ -30,7 +30,7 @@ @Test(groups = "flaky") public class TransactionEndToEndWithoutBatchIndexAckTest extends TransactionEndToEndTest { - @BeforeMethod + @BeforeClass protected void setup() throws Exception { conf.setAcknowledgmentAtBatchIndexLevelEnabled(false); setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java index 795cf2b7f7cd7..85f6f16fdb01a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.compaction; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertTrue; import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; @@ -25,6 +28,13 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.util.Arrays; +import java.util.Optional; +import java.util.Properties; +import lombok.Cleanup; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.util.CmdGenerateDocs; import org.testng.annotations.Test; @@ -72,4 +82,44 @@ public void testGenerateDocs() throws Exception { System.setOut(oldStream); } } + + @Test + public void testUseTlsUrlWithPEM() throws PulsarClientException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setBrokerServicePortTls(Optional.of(6651)); + serviceConfiguration.setBrokerClientTlsEnabled(true); + serviceConfiguration.setProperties(new Properties()); + + @Cleanup + PulsarClient ignored = CompactorTool.createClient(serviceConfiguration); + + verify(serviceConfiguration, times(1)).isBrokerClientTlsEnabled(); + verify(serviceConfiguration, times(1)).isTlsAllowInsecureConnection(); + verify(serviceConfiguration, times(1)).getBrokerClientKeyFilePath(); + verify(serviceConfiguration, times(1)).getBrokerClientTrustCertsFilePath(); + verify(serviceConfiguration, times(1)).getBrokerClientCertificateFilePath(); + } + + @Test + public void testUseTlsUrlWithKeystore() throws PulsarClientException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setBrokerServicePortTls(Optional.of(6651)); + serviceConfiguration.setBrokerClientTlsEnabled(true); + serviceConfiguration.setBrokerClientTlsEnabledWithKeyStore(true); + serviceConfiguration.setBrokerClientTlsTrustStore(MockedPulsarServiceBaseTest.BROKER_KEYSTORE_FILE_PATH); + + serviceConfiguration.setProperties(new Properties()); + + @Cleanup + PulsarClient ignored = CompactorTool.createClient(serviceConfiguration); + + verify(serviceConfiguration, times(1)).isBrokerClientTlsEnabled(); + verify(serviceConfiguration, times(1)).isBrokerClientTlsEnabledWithKeyStore(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsKeyStore(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsKeyStorePassword(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsKeyStoreType(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsTrustStore(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsTrustStorePassword(); + verify(serviceConfiguration, times(1)).getBrokerClientTlsTrustStoreType(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java index 93d1132d812d0..af72a2564bb0c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java @@ -135,6 +135,7 @@ public void testTopicInternalStats() throws Exception { && (cursor.totalNonContiguousDeletedMessagesRange) < numberOfMsgs / 2); assertFalse(cursor.subscriptionHavePendingRead); assertFalse(cursor.subscriptionHavePendingReplayRead); + assertTrue(cursor.active); producer.close(); consumer.close(); log.info("-- Exiting {} test --", methodName); diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java index 52a86e23f06be..6ccb442c70487 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java @@ -319,7 +319,7 @@ public void setConcurrentLookupRequest(int concurrentLookupRequest) { } /** - * Get configured max number of reject-request in a time-frame (30 seconds) after which connection will be closed. + * Get configured max number of reject-request in a time-frame (60 seconds) after which connection will be closed. * * @return */ @@ -328,7 +328,7 @@ public int getMaxNumberOfRejectedRequestPerConnection() { } /** - * Set max number of broker-rejected requests in a certain time-frame (30 seconds) after which current connection. + * Set max number of broker-rejected requests in a certain time-frame (60 seconds) after which current connection. * will be closed and client creates a new connection that give chance to connect a different broker (default: * 50) * diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index 177cae9a9a438..d2d0754f1d112 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -4397,4 +4397,58 @@ CompletableFuture> examineMessageAsync(String topic, String init * @param sourceTopic source topic name */ CompletableFuture> getShadowTopicsAsync(String sourceTopic); + + /** + * Get the shadow source topic name of the given shadow topic. + * @param shadowTopic shadow topic name. + * @return The topic name of the source of the shadow topic. + */ + String getShadowSource(String shadowTopic) throws PulsarAdminException; + + /** + * Get the shadow source topic name of the given shadow topic asynchronously. + * @param shadowTopic shadow topic name. + * @return The topic name of the source of the shadow topic. + */ + CompletableFuture getShadowSourceAsync(String shadowTopic); + + /** + * Create a new shadow topic as the shadow of the source topic. + * The source topic must exist before call this method. + *

+ * For partitioned source topic, the partition number of shadow topic follows the source topic at creation. If + * the partition number of the source topic changes, the shadow topic needs to update its partition number + * manually. + * For non-partitioned source topic, the shadow topic will be created as non-partitioned topic. + *

+ * + * NOTE: This is still WIP until PIP-180 is finished. + * + * @param shadowTopic shadow topic name, and it must be a persistent topic name. + * @param sourceTopic source topic name, and it must be a persistent topic name. + * @param properties properties to be created with in the shadow topic. + * @throws PulsarAdminException + */ + void createShadowTopic(String shadowTopic, String sourceTopic, Map properties) + throws PulsarAdminException; + + /** + * Create a new shadow topic, see #{@link #createShadowTopic(String, String, Map)} for details. + */ + CompletableFuture createShadowTopicAsync(String shadowTopic, String sourceTopic, + Map properties); + + /** + * Create a new shadow topic, see #{@link #createShadowTopic(String, String, Map)} for details. + */ + default void createShadowTopic(String shadowTopic, String sourceTopic) throws PulsarAdminException { + createShadowTopic(shadowTopic, sourceTopic, null); + } + + /** + * Create a new shadow topic, see #{@link #createShadowTopic(String, String, Map)} for details. + */ + default CompletableFuture createShadowTopicAsync(String shadowTopic, String sourceTopic) { + return createShadowTopicAsync(shadowTopic, sourceTopic, null); + } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index 35bef4cda33c6..ee67f826ee678 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -94,6 +94,7 @@ public static class CursorStats { public String individuallyDeletedMessages; public String lastLedgerSwitchTimestamp; public String state; + public boolean active; public long numberOfEntriesSinceFirstNotAckedMessage; public int totalNonContiguousDeletedMessagesRange; public boolean subscriptionHavePendingRead; diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index e3b51accdfd7c..4312080ac2233 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -131,6 +131,8 @@ public class TopicsImpl extends BaseResource implements Topics { private static final String ENCRYPTION_KEYS = "X-Pulsar-Base64-encryption-keys"; // CHECKSTYLE.ON: MemberName + public static final String PROPERTY_SHADOW_SOURCE_KEY = "PULSAR.SHADOW_SOURCE"; + public TopicsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { super(auth, readTimeoutMs); adminTopics = web.path("/admin"); @@ -2705,7 +2707,43 @@ public CompletableFuture removeShadowTopicsAsync(String sourceTopic) { public CompletableFuture> getShadowTopicsAsync(String sourceTopic) { TopicName tn = validateTopic(sourceTopic); WebTarget path = topicPath(tn, "shadowTopics"); - return asyncGetRequest(path, new FutureCallback>(){}); + return asyncGetRequest(path, new FutureCallback>() {}); + } + + @Override + public String getShadowSource(String shadowTopic) throws PulsarAdminException { + return sync(() -> getShadowSourceAsync(shadowTopic)); + } + + @Override + public CompletableFuture getShadowSourceAsync(String shadowTopic) { + return getPropertiesAsync(shadowTopic).thenApply( + properties -> properties != null ? properties.get(PROPERTY_SHADOW_SOURCE_KEY) : null); + } + + @Override + public void createShadowTopic(String shadowTopic, String sourceTopic, Map properties) + throws PulsarAdminException { + sync(() -> createShadowTopicAsync(shadowTopic, sourceTopic, properties)); + } + + @Override + public CompletableFuture createShadowTopicAsync(String shadowTopic, String sourceTopic, + Map properties) { + checkArgument(TopicName.get(shadowTopic).isPersistent(), "Shadow topic must be persistent"); + checkArgument(TopicName.get(sourceTopic).isPersistent(), "Source topic must be persistent"); + return getPartitionedTopicMetadataAsync(sourceTopic).thenCompose(sourceTopicMeta -> { + HashMap shadowProperties = new HashMap<>(); + if (properties != null) { + shadowProperties.putAll(properties); + } + shadowProperties.put(PROPERTY_SHADOW_SOURCE_KEY, sourceTopic); + if (sourceTopicMeta.partitions == PartitionedTopicMetadata.NON_PARTITIONED) { + return createNonPartitionedTopicAsync(shadowTopic, shadowProperties); + } else { + return createPartitionedTopicAsync(shadowTopic, sourceTopicMeta.partitions, shadowProperties); + } + }); } private static final Logger log = LoggerFactory.getLogger(TopicsImpl.class); diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 6645488d70471..1adaafbb90dff 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -397,6 +397,33 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + enforce-bytecode-version + + enforce + + + + + ${pulsar.client.compiler.release} + + + + + + + + org.codehaus.mojo + extra-enforcer-rules + ${extra-enforcer-rules.version} + + + diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java index 6d44f20b8eef4..17852bc545dc0 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java @@ -489,7 +489,7 @@ ClientBuilder authentication(String authPluginClassName, Map aut ClientBuilder maxLookupRedirects(int maxLookupRedirects); /** - * Set max number of broker-rejected requests in a certain time-frame (30 seconds) after which current connection + * Set max number of broker-rejected requests in a certain time-frame (60 seconds) after which current connection * will be closed and client creates a new connection that give chance to connect a different broker (default: * 50). * diff --git a/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java b/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java index a10bee1711c30..8c719b861b0bc 100644 --- a/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java +++ b/pulsar-client-auth-athenz/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationAthenz.java @@ -65,7 +65,7 @@ public class AuthenticationAthenz implements Authentication, EncodedAuthenticati // ZTSClient.cancelPrefetch() is called. // cf. https://github.com/AthenZ/athenz/issues/544 private boolean autoPrefetchEnabled = false; - private long cachedRoleTokenTimestamp; + private volatile long cachedRoleTokenTimestamp; private String roleToken; // athenz will only give this token if it's at least valid for 2hrs private static final int minValidity = 2 * 60 * 60; diff --git a/pulsar-client-cpp/.clang-format b/pulsar-client-cpp/.clang-format deleted file mode 100644 index cb40b5069841e..0000000000000 --- a/pulsar-client-cpp/.clang-format +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -BasedOnStyle: Google -IndentWidth: 4 -ColumnLimit: 110 -SortIncludes: false -BreakBeforeBraces: Custom -BraceWrapping: - AfterEnum: true diff --git a/pulsar-client-cpp/.gitignore b/pulsar-client-cpp/.gitignore deleted file mode 100644 index 8c8c065e61935..0000000000000 --- a/pulsar-client-cpp/.gitignore +++ /dev/null @@ -1,91 +0,0 @@ -# Compiled Object files -*.slo -*.lo -*.o -*.obj -*.os -*.scons* - -# Compiled Dynamic libraries -*.so -lib*.so* -*.dylib -*.dll -*.so.1 - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# Dependency file -*.d - -# Mac swap file -*.DS_Store - -# Linux swap file -*.swp - -# Exclude compiled executables -/examples/SampleProducer -/examples/SampleProducerCApi -/examples/SampleConsumer -/examples/SampleConsumerCApi -/examples/SampleAsyncProducer -/examples/SampleAsyncConsumerCApi -/examples/SampleConsumerListener -/examples/SampleConsumerListenerCApi -/examples/SampleReaderCApi -/examples/SampleFileLogger -/tests/main -/perf/perfProducer -/perf/perfConsumer -/system-test/SystemTest - -# Files generated from templates by CMAKE -include/pulsar/Version.h - -# IDE generated files -.csettings -.cproject -.project -.settings/ -.pydevproject -.idea/ -.vs/ -*.cbp -*.ninja* -.clangd/ -compile_commands.json - -# doxygen files -apidocs/ - -# generated files -generated/ - -# CMAKE -.cmake -Makefile -cmake_install.cmake -CMakeFiles -CMakeCache.txt - -pulsar-dist -install_manifest.txt -merged-library -python/venv - -# Visual Studio files -out/ -CMakeSettings.json - -# vcpkg dependencies directory -vcpkg_installed/ diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt deleted file mode 100644 index 1975cd64bf7e1..0000000000000 --- a/pulsar-client-cpp/CMakeLists.txt +++ /dev/null @@ -1,465 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cmake_minimum_required(VERSION 3.4) - -project (pulsar-cpp) -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") - -execute_process(COMMAND ${CMAKE_SOURCE_DIR}/../src/gen-pulsar-version-macro.py OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE PVM) -set(PVM_COMMENT "This is generated from Version.h.in by CMAKE. DO NOT EDIT DIRECTLY") -configure_file(templates/Version.h.in include/pulsar/Version.h @ONLY) - -if (VCPKG_TRIPLET) - message(STATUS "Use vcpkg, triplet is ${VCPKG_TRIPLET}") - set(CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") - message(STATUS "Use CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}") - set(PROTOC_PATH "${CMAKE_PREFIX_PATH}/tools/protobuf/protoc") - message(STATUS "Use protoc: ${PROTOC_PATH}") - set(VCPKG_DEBUG_ROOT "${CMAKE_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}/debug") - if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(ZLIB_ROOT ${VCPKG_DEBUG_ROOT}) - set(OPENSSL_ROOT_DIR ${VCPKG_DEBUG_ROOT}) - endif () -endif() - -find_program(CCACHE_PROGRAM ccache) -if(CCACHE_PROGRAM) - set(CMAKE_CXX_COMPILER_LAUNCHER "ccache") - MESSAGE(STATUS "Using CCache") -endif(CCACHE_PROGRAM) - -MESSAGE(STATUS "ARCHITECTURE: ${CMAKE_SYSTEM_PROCESSOR}") - -option(BUILD_DYNAMIC_LIB "Build dynamic lib" ON) -MESSAGE(STATUS "BUILD_DYNAMIC_LIB: " ${BUILD_DYNAMIC_LIB}) - -option(BUILD_STATIC_LIB "Build static lib" ON) -MESSAGE(STATUS "BUILD_STATIC_LIB: " ${BUILD_STATIC_LIB}) - -option(BUILD_TESTS "Build tests" ON) -MESSAGE(STATUS "BUILD_TESTS: " ${BUILD_TESTS}) - -option(BUILD_PYTHON_WRAPPER "Build Pulsar Python wrapper" ON) -MESSAGE(STATUS "BUILD_PYTHON_WRAPPER: " ${BUILD_PYTHON_WRAPPER}) - -option(BUILD_WIRESHARK "Build Pulsar Wireshark dissector" OFF) -MESSAGE(STATUS "BUILD_WIRESHARK: " ${BUILD_WIRESHARK}) - -option(BUILD_PERF_TOOLS "Build Pulsar CLI perf producer/consumer" OFF) -MESSAGE(STATUS "BUILD_PERF_TOOLS: " ${BUILD_PERF_TOOLS}) - -option(LINK_STATIC "Link against static libraries" OFF) -MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) - -option(USE_LOG4CXX "Build with Log4cxx support" OFF) -MESSAGE(STATUS "USE_LOG4CXX: " ${USE_LOG4CXX}) - -IF (CMAKE_BUILD_TYPE STREQUAL "") - set(CMAKE_BUILD_TYPE RelWithDebInfo) -ENDIF () - -MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) - -set(THREADS_PREFER_PTHREAD_FLAG TRUE) -find_package(Threads REQUIRED) -MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) - -set(Boost_NO_BOOST_CMAKE ON) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_C_STANDARD 11) - -# Compiler specific configuration: -# https://stackoverflow.com/questions/10046114/in-cmake-how-can-i-test-if-the-compiler-is-clang -if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - add_definitions(-DWIN32_LEAN_AND_MEAN -DNOGDI -D_WIN32_WINNT=0x0501 -D_CRT_SECURE_NO_WARNINGS) - add_compile_options(/wd4244 /wd4267 /wd4018 /wd4715 /wd4251 /wd4275) -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - # ?? Don't have this to test with -else() # GCC or Clang are mostly compatible: - # Turn on warnings and enable warnings-as-errors: - add_compile_options(-Wall -Wformat-security -Wvla -Werror) - # Turn off certain warnings that are too much pain for too little gain: - add_compile_options(-Wno-sign-compare -Wno-deprecated-declarations -Wno-error=cpp) - if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR APPLE) - add_compile_options(-msse4.2 -mpclmul) - endif() - # Options unique to Clang or GCC: - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options(-Qunused-arguments) - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.1)) - add_compile_options(-Wno-stringop-truncation) - endif() -endif() - -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -set(LOG_CATEGORY_NAME $ENV{LOG_CATEGORY_NAME}) - -if (NOT LOG_CATEGORY_NAME) - set(LOG_CATEGORY_NAME "\"pulsar.\"") -endif(NOT LOG_CATEGORY_NAME) - -add_definitions(-DLOG_CATEGORY_NAME=${LOG_CATEGORY_NAME} -DBUILDING_PULSAR -DBOOST_ALL_NO_LIB -DBOOST_ALLOW_DEPRECATED_HEADERS) - -set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/) - -### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY -### After find the libs, will unset related cache, and will not affect another same call to find_package. -if (APPLE) - set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include) - set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl) -endif () - -set(OPENSSL_USE_STATIC_LIBS FALSE) -find_package(OpenSSL REQUIRED) -set(RECORD_OPENSSL_SSL_LIBRARY ${OPENSSL_SSL_LIBRARY}) -set(RECORD_OPENSSL_CRYPTO_LIBRARY ${OPENSSL_CRYPTO_LIBRARY}) - -unset(OPENSSL_FOUND CACHE) -unset(OPENSSL_INCLUDE_DIR CACHE) -unset(OPENSSL_CRYPTO_LIBRARY CACHE) -unset(OPENSSL_CRYPTO_LIBRARIES CACHE) -unset(OPENSSL_SSL_LIBRARY CACHE) -unset(OPENSSL_SSL_LIBRARIES CACHE) -unset(OPENSSL_LIBRARIES CACHE) -unset(OPENSSL_VERSION CACHE) - -if (LINK_STATIC) - find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib) - message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}") - find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf) - message(STATUS "Protobuf: ${Protobuf_LIBRARIES}") - find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a) - message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}") - find_library(LIB_ZSTD NAMES libzstd.a) - message(STATUS "ZStd: ${LIB_ZSTD}") - find_library(LIB_SNAPPY NAMES libsnappy.a) - message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") - set(COMMON_LIBS ${Protobuf_LIBRARIES} ${COMMON_LIBS}) - - if (USE_LOG4CXX) - if (LOG4CXX_USE_DYNAMIC_LIBS) - find_library(LOG4CXX_LIBRARY_PATH log4cxx) - else () - find_library(LOG4CXX_LIBRARY_PATH NAMES liblog4cxx.a) - - # Libraries needed by log4cxx to link statically with - find_library(APR_LIBRARY_PATH NAMES libapr-1 PATHS /usr/lib /usr/local/apr/lib /usr/local/opt/apr/libexec/lib/) - find_library(APR_UTIL_LIBRARY_PATH NAMES libaprutil-1 PATHS /usr/lib /usr/local/apr/lib /usr/local/opt/apr-util/libexec/lib/) - find_library(EXPAT_LIBRARY_PATH NAMES libexpat expat) - if (APPLE) - find_library(ICONV_LIBRARY_PATH NAMES libiconv iconv) - else () - set(ICONV_LIBRARY_PATH ) - endif (APPLE) - endif (LOG4CXX_USE_DYNAMIC_LIBS) - endif (USE_LOG4CXX) - - if (MSVC) - add_definitions(-DCURL_STATICLIB) - endif() - - if (UNIX AND NOT APPLE) - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) - endif() - - SET(Boost_USE_STATIC_LIBS ON) - SET(OPENSSL_USE_STATIC_LIBS TRUE) -else() - # Link to shared libraries - find_package(ZLIB REQUIRED) - set(ZLIB_LIBRARIES ${ZLIB_LIBRARIES}) - # NOTE: The default MODULE mode may not find debug libraries so use CONFIG mode here - unset(Protobuf_INCLUDE_DIRS CACHE) - unset(Protobuf_LIBRARIES CACHE) - find_package(Protobuf QUIET CONFIG) - # NOTE: On Windows x86 platform, Protobuf_FOUND might be set false but Protobuf_INCLUDE_DIRS and - # Protobuf_LIBRARIES are both found. - if (Protobuf_INCLUDE_DIRS AND Protobuf_LIBRARIES AND NOT Protobuf_FOUND) - set(Protobuf_FOUND TRUE) - endif () - if (Protobuf_FOUND) - message("Found Protobuf in config mode") - message(STATUS "Protobuf_LIBRARIES: ${Protobuf_LIBRARIES}") - message(STATUS "Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS}") - else () - message("Failed to find Protobuf in config mode, try to find it from system path") - find_library(Protobuf_LIBRARIES protobuf libprotobuf) - find_path(Protobuf_INCLUDE_DIRS google/protobuf/stubs/common.h) - message(STATUS "Protobuf_LIBRARIES: ${Protobuf_LIBRARIES}") - message(STATUS "Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS}") - endif () - - if (${Protobuf_FOUND} AND (${CMAKE_VERSION} VERSION_GREATER 3.8)) - set(COMMON_LIBS protobuf::libprotobuf ${COMMON_LIBS}) - else () - set(COMMON_LIBS ${Protobuf_LIBRARIES} ${COMMON_LIBS}) - endif () - - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(LIB_ZSTD zstdd HINTS "${VCPKG_DEBUG_ROOT}/lib") - else () - find_library(LIB_ZSTD zstd) - endif () - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(LIB_SNAPPY NAMES snappyd HINTS "${VCPKG_DEBUG_ROOT}/lib") - else () - find_library(LIB_SNAPPY NAMES snappy libsnappy) - endif () - - find_package(CURL REQUIRED) - if (${CMAKE_VERSION} VERSION_GREATER "3.12") - set(COMMON_LIBS ${COMMON_LIBS} CURL::libcurl) - endif () - - if (USE_LOG4CXX) - find_library(LOG4CXX_LIBRARY_PATH log4cxx) - find_path(LOG4CXX_INCLUDE_PATH log4cxx/logger.h) - endif (USE_LOG4CXX) -endif (LINK_STATIC) - - -find_package(Boost) - -if (Boost_MAJOR_VERSION EQUAL 1 AND Boost_MINOR_VERSION LESS 69) - # Boost System does not require linking since 1.69 - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system) - MESSAGE(STATUS "Linking with Boost:System") -endif() - -if (MSVC) - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} date_time) -endif() - -if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) - # GCC 4.8.2 implementation of std::regex is buggy - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) - set(CMAKE_CXX_FLAGS " -DPULSAR_USE_BOOST_REGEX") - MESSAGE(STATUS "Using Boost::Regex") -else() - MESSAGE(STATUS "Using std::regex") - # Turn on color error messages and show additional help with errors (only available in GCC v4.9+): - add_compile_options(-fdiagnostics-show-option -fdiagnostics-color) -endif() - -if(BUILD_PERF_TOOLS) - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} program_options) -endif() - -find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) - -if (BUILD_PYTHON_WRAPPER) - find_package(PythonLibs REQUIRED) - MESSAGE(STATUS "PYTHON: " ${PYTHONLIBS_VERSION_STRING}) - - string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${PYTHONLIBS_VERSION_STRING}) - list(GET PYTHONLIBS_VERSION_NO_LIST 0 PYTHONLIBS_VERSION_MAJOR) - list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR) - set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR}) - # For python3 the lib name is boost_python3 - set(BOOST_PYTHON_NAME_LIST python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX}) - - foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST) - find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME}) - if (${Boost_FOUND}) - set(BOOST_PYTHON_NAME_FOUND ${BOOST_PYTHON_NAME}) - break() - endif() - endforeach() - - if (NOT ${Boost_FOUND}) - MESSAGE(FATAL_ERROR "Could not find Boost Python library") - endif () - - MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND}) - find_package(Boost REQUIRED COMPONENTS ${BOOST_PYTHON_NAME_FOUND}) -endif (BUILD_PYTHON_WRAPPER) - -find_package(OpenSSL REQUIRED) - -if (BUILD_TESTS) - find_path(GTEST_INCLUDE_PATH gtest/gtest.h) - find_path(GMOCK_INCLUDE_PATH gmock/gmock.h) -endif () - -if (USE_LOG4CXX) - set(CMAKE_CXX_FLAGS " -DUSE_LOG4CXX ${CMAKE_CXX_FLAGS}") - find_path(LOG4CXX_INCLUDE_PATH log4cxx/logger.h) -endif (USE_LOG4CXX) - -if (NOT APPLE AND NOT MSVC) - # we don't set options below to build _pulsar.so - set(CMAKE_CXX_FLAGS_PYTHON "${CMAKE_CXX_FLAGS}") - # Hide all non-exported symbols to avoid conflicts - add_compile_options(-fvisibility=hidden) - if (CMAKE_COMPILER_IS_GNUCC) - add_compile_options(-Wl,--exclude-libs,ALL) - endif () -endif () - -if (LIB_ZSTD) - set(HAS_ZSTD 1) -else () - set(HAS_ZSTD 0) -endif () -MESSAGE(STATUS "HAS_ZSTD: ${HAS_ZSTD}") - -if (LIB_SNAPPY) - set(HAS_SNAPPY 1) -else () - set(HAS_SNAPPY 0) -endif () -MESSAGE(STATUS "HAS_SNAPPY: ${HAS_SNAPPY}") - -set(ADDITIONAL_LIBRARIES $ENV{PULSAR_ADDITIONAL_LIBRARIES}) -link_directories( $ENV{PULSAR_ADDITIONAL_LIBRARY_PATH} ) - -set(AUTOGEN_DIR ${CMAKE_BINARY_DIR}/generated) -file(MAKE_DIRECTORY ${AUTOGEN_DIR}) - -include_directories( - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_BINARY_DIR}/include - ${AUTOGEN_DIR} - ${Boost_INCLUDE_DIR} - ${OPENSSL_INCLUDE_DIR} - ${ZLIB_INCLUDE_DIRS} - ${CURL_INCLUDE_DIRS} - ${Protobuf_INCLUDE_DIRS} - ${LOG4CXX_INCLUDE_PATH} - ${GTEST_INCLUDE_PATH} - ${GMOCK_INCLUDE_PATH} -) - -set(COMMON_LIBS - ${COMMON_LIBS} - Threads::Threads - ${Boost_REGEX_LIBRARY} - ${Boost_SYSTEM_LIBRARY} - ${CURL_LIBRARIES} - ${OPENSSL_LIBRARIES} - ${ZLIB_LIBRARIES} - ${ADDITIONAL_LIBRARIES} - ${CMAKE_DL_LIBS} -) - -if (MSVC) - set(COMMON_LIBS - ${COMMON_LIBS} - ${Boost_DATE_TIME_LIBRARY} - ) -endif() - -if (NOT MSVC) - set(COMMON_LIBS ${COMMON_LIBS} m) -else() - set(COMMON_LIBS - ${COMMON_LIBS} - wldap32.lib - Normaliz.lib) -endif() - -if (USE_LOG4CXX) - set(COMMON_LIBS - ${COMMON_LIBS} - ${LOG4CXX_LIBRARY_PATH} - ${APR_LIBRARY_PATH} - ${APR_UTIL_LIBRARY_PATH} - ${EXPAT_LIBRARY_PATH} - ${ICONV_LIBRARY_PATH} - ) -endif () - -if (HAS_ZSTD) - set(COMMON_LIBS ${COMMON_LIBS} ${LIB_ZSTD} ) -endif () - -add_definitions(-DHAS_ZSTD=${HAS_ZSTD}) - -if (HAS_SNAPPY) - set(COMMON_LIBS ${COMMON_LIBS} ${LIB_SNAPPY} ) -endif () - -add_definitions(-DHAS_SNAPPY=${HAS_SNAPPY}) - -if(NOT APPLE AND NOT MSVC) - set(COMMON_LIBS ${COMMON_LIBS} rt) -endif () - -link_directories(${CMAKE_BINARY_DIR}/lib) - -set(LIB_NAME $ENV{PULSAR_LIBRARY_NAME}) -if (NOT LIB_NAME) - set(LIB_NAME pulsar) -endif(NOT LIB_NAME) - -set(CLIENT_LIBS - ${COMMON_LIBS} - ${LIB_NAME} -) - -add_subdirectory(lib) -if(BUILD_PERF_TOOLS) - add_subdirectory(perf) -endif(BUILD_PERF_TOOLS) - -if (BUILD_DYNAMIC_LIB) - add_subdirectory(examples) -endif() - -if (BUILD_TESTS) - add_subdirectory(tests) -endif() - -if (BUILD_PYTHON_WRAPPER) - add_subdirectory(python) -endif () - -if (BUILD_WIRESHARK) - add_subdirectory(wireshark) -endif() - -find_package(ClangTools) -set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") -add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py - ${CLANG_FORMAT_BIN} - 0 - ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt - ${CMAKE_SOURCE_DIR}/lib - ${CMAKE_SOURCE_DIR}/perf - ${CMAKE_SOURCE_DIR}/examples - ${CMAKE_SOURCE_DIR}/tests - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/python/src - ${CMAKE_SOURCE_DIR}/wireshark) - -# `make check-format` option (for CI test) -add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py - ${CLANG_FORMAT_BIN} - 1 - ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt - ${CMAKE_SOURCE_DIR}/lib - ${CMAKE_SOURCE_DIR}/perf - ${CMAKE_SOURCE_DIR}/examples - ${CMAKE_SOURCE_DIR}/tests - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/python/src - ${CMAKE_SOURCE_DIR}/wireshark) diff --git a/pulsar-client-cpp/Doxyfile b/pulsar-client-cpp/Doxyfile deleted file mode 100644 index b3ac004d9a89b..0000000000000 --- a/pulsar-client-cpp/Doxyfile +++ /dev/null @@ -1,2500 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Doxyfile 1.8.14 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See -# https://www.gnu.org/software/libiconv/ for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = "pulsar-client-cpp" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../target/doxygen - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:\n" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 0 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = include docs/MainPage.md - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = docs/MainPage.md - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see https://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# https://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML -# documentation will contain a main index with vertical navigation menus that -# are dynamically created via Javascript. If disabled, the navigation index will -# consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have Javascript, -# like the Qt help browser. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_MENUS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See https://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: http://doc.qt.io/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://doc.qt.io/qt-4.8/qthelpproject.html#virtual-folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://doc.qt.io/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side Javascript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /