Blue green

.asf.yaml

@@ -87,5 +87,5 @@ notifications:
   commits:      commits@pulsar.apache.org
   issues:       commits@pulsar.apache.org
   pullrequests: commits@pulsar.apache.org
-  discussions:  dev@pulsar.apache.org
+  discussions:  commits@pulsar.apache.org
   jira_options: link label

.github/ISSUE_TEMPLATE/flaky-test.yml

@@ -28,7 +28,7 @@ body:
     attributes:
       label: Search before asking
       description: >
-        Please search [issues](https://github.com/apache/pulsar/issues) to check if your issue has already been reported.
+        Please search [issues](https://github.com/apache/pulsar/issues) to check if your issue has already been reported. First search with the test method name and then with the test class name to see if there are reports for the same test method or the same test class.
       options:
         - label: >
             I searched in the [issues](https://github.com/apache/pulsar/issues) and found nothing similar.
@@ -45,16 +45,12 @@ body:
     attributes:
       label: Exception stacktrace
       description: |
-        A few lines of the stack trace that shows at least the exception message and the line of test code where the stacktrace occurred.
+        Copy-paste the stack trace from the build log. If the stacktrace is >100 lines, you can limit the stack trace to the point where it includes the stack frame for the test method so that it's possible to find out where the exception occurred in the test.
       value: |
-        <!-- optionally provide the full stacktrace -->
+        <!-- copy-paste the stack trace in the code block below -->
+        ```
 
-        <details>
-        <summary>Full exception stacktrace</summary>
-        <pre><code>
-          full exception stacktrace here
-        </code></pre>
-        </details>
+        ```
     validations:
       required: true
   - type: checkboxes

.github/ISSUE_TEMPLATE/pip.yml

@@ -60,6 +60,15 @@ body:
         This should also serve as documentation for any person that is trying to understand or debug the behavior of a certain feature.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: Security Considerations
+      description: |
+        A detailed description of the security details that ought to be considered for the PIP. This is most relevant for any new HTTP endpoints, new Pulsar Protocol Commands, and new security features. The goal is to describe details like which role will have permission to perform an action.
+
+        If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list.
+    validations:
+      required: true
   - type: textarea
     attributes:
       label: Alternatives

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 <!--
 ### Contribution Checklist
 
-  - PR title format should be *[type][component] summary*. For details, see *[Guideline - Pulsar PR Naming Convention](https://docs.google.com/document/d/1d8Pw6ZbWk-_pCKdOmdvx9rnhPiyuxwq60_TrD68d7BA/edit#heading=h.trs9rsex3xom)*. 
+  - PR title format should be *[type][component] summary*. For details, see *[Guideline - Pulsar PR Naming Convention](https://pulsar.apache.org/contribute/develop-semantic-title/)*. 
 
   - Fill out the template below to describe the changes contributed by the pull request. That will give reviewers the context they need to do the review.
 
@@ -20,6 +20,12 @@ Fixes #xyz
 
 Master Issue: #xyz
 
+<!-- If the PR belongs to a PIP, please add the PIP link here -->
+
+PIP: #xyz 
+
+<!-- Details of when a PIP is required and how the PIP process work, please see: https://github.com/apache/pulsar/blob/master/wiki/proposals/PIP.md -->
+
 ### Motivation
 
 <!-- Explain here the context, and why you're making that change. What is the problem you're trying to solve. -->
@@ -50,6 +56,8 @@ This change added tests and can be verified as follows:
 
 ### Does this pull request potentially affect one of the following parts:
 
+<!-- DO NOT REMOVE THIS SECTION. CHECK THE PROPER BOX ONLY. -->
+
 *If the box was checked, please highlight the changes*
 
 - [ ] Dependencies (add or upgrade a dependency)
@@ -60,13 +68,14 @@ This change added tests and can be verified as follows:
 - [ ] The binary protocol
 - [ ] The REST endpoints
 - [ ] The admin CLI options
+- [ ] The metrics
 - [ ] Anything that affects deployment
 
 ### Documentation
 
 <!-- DO NOT REMOVE THIS SECTION. CHECK THE PROPER BOX ONLY. -->
 
-- [ ] `doc` <!-- Your PR contains doc changes. Please attach the local preview screenshots (run `sh start.sh` at `pulsar/site2/website`) to your PR description, or else your PR might not get merged. -->
+- [ ] `doc` <!-- Your PR contains doc changes. -->
 - [ ] `doc-required` <!-- Your PR changes impact docs and you will update later -->
 - [ ] `doc-not-needed` <!-- Your PR changes do not impact docs -->
 - [ ] `doc-complete` <!-- Docs have been already added -->

.github/actions/gradle-enterprise/action.yml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Configure Gradle Enterprise integration
+description: Configure Gradle Enterprise when secret GE_ACCESS_TOKEN is available
+inputs:
+  token:
+    description: 'The token for accessing Gradle Enterprise'
+    required: true
+runs:
+  using: composite
+  steps:
+    - run: |
+        if [[ -n "${{ inputs.token }}" ]]; then
+            echo "::group::Configuring Gradle Enterprise for build"
+            cp .mvn/ge-extensions.xml .mvn/extensions.xml
+            echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV
+            echo "::endgroup::"  
+        fi
+      shell: bash

.github/actions/ssh-access/action.yml

@@ -121,8 +121,22 @@ runs:
             tmux set -t upterm window-size largest
             echo '::endgroup::'  
             echo -e "\nSSH connection information"
+            # wait up to 10 seconds for upterm admin socket to appear
+            for i in {1..10}; do
+              ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
+              if [ ! -S "$ADMIN_SOCKET" ]; then
+                echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
+                sleep 1
+              else
+                echo "upterm admin socket available in $ADMIN_SOCKET"
+                break
+              fi
+            done
             shopt -s nullglob
-            upterm session current --admin-socket ~/.upterm/*.sock
+            upterm session current --admin-socket ~/.upterm/*.sock || {
+              echo "Starting upterm failed."
+              exit 0
+            }
         elif [[ "${{ inputs.action }}" == "wait" ]]; then
             # only wait if upterm was installed
             if command -v upterm &>/dev/null; then

.github/actions/upload-coverage/action.yml

@@ -0,0 +1,100 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Upload to Codecov with retries
+description: |
+  Checks that the current repository is public and then
+  uploads to codecov with multiple retries as a workaround 
+  for these issues
+    - https://github.com/codecov/codecov-action/issues/598
+    - https://github.com/codecov/codecov-action/issues/837
+inputs:
+  flags:
+    # see https://github.com/codecov/codecov-action#arguments
+    description: 'Flag the upload to group coverage metrics. Multiple flags are separated by a comma.'
+runs:
+  using: composite
+  steps:
+    - name: "Check that repository is public"
+      id: repo-check
+      shell: bash
+      run: |
+        if [[ "${{ github.server_url }}" != "https://github.com" ]]; then
+          echo "Not using github.com server ('${{ github.server_url }}'). Skipping uploading of coverage metrics."
+          echo "passed=false" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        REPO_URL="${{ github.server_url }}/${{ github.repository }}"
+        {
+          # public repository url will respond to http HEAD request
+          curl -X HEAD -fs "$REPO_URL" && echo "passed=true" >> $GITHUB_OUTPUT
+        } || {
+          echo "$REPO_URL isn't a public repository. Skipping uploading of coverage metrics."
+          echo "passed=false" >> $GITHUB_OUTPUT
+        }
+    - name: "Upload to Codecov (attempt #1)"
+      id: codecov-upload-1
+      if: steps.repo-check.outputs.passed == 'true'
+      uses: codecov/codecov-action@v3
+      continue-on-error: true
+      with:
+        flags: ${{ inputs.flags }}
+        fail_ci_if_error: true
+        verbose: true
+    - name: "Wait 15 seconds before next attempt"
+      if: steps.codecov-upload-1.outcome == 'failure'
+      shell: bash
+      run: sleep 15
+    - name: "Upload to Codecov (attempt #2)"
+      id: codecov-upload-2
+      if: steps.codecov-upload-1.outcome == 'failure'
+      uses: codecov/codecov-action@v3
+      continue-on-error: true
+      with:
+        flags: ${{ inputs.flags }}
+        fail_ci_if_error: true
+        verbose: true
+    - name: "Wait 60 seconds before next attempt"
+      if: steps.codecov-upload-2.outcome == 'failure'
+      shell: bash
+      run: sleep 60
+    - name: "Upload to Codecov (attempt #3)"
+      id: codecov-upload-3
+      if: steps.codecov-upload-2.outcome == 'failure'
+      uses: codecov/codecov-action@v3
+      # fail on last attempt
+      continue-on-error: false
+      with:
+        flags: ${{ inputs.flags }}
+        fail_ci_if_error: true
+        verbose: true
+    - name: "Show link to Codecov report"
+      shell: bash
+      run: |
+        if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+          head_sha=$(jq -r '.pull_request.head.sha' "${GITHUB_EVENT_PATH}")
+        else
+          head_sha=$(git rev-parse HEAD)
+        fi
+        tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+        ## Code Coverage report in Codecov
+
+        [Code coverage report](https://app.codecov.io/github/$GITHUB_REPOSITORY/commit/${head_sha}/tree)
+
+        EOF

.github/changes-filter.yaml

@@ -3,13 +3,13 @@
 all:
   - '**'
 docs:
-  - 'site2/**'
-  - 'deployment/**'
-  - '.asf.yaml'
   - '*.md'
   - '**/*.md'
+  - '.asf.yaml'
   - '.github/changes-filter.yaml'
   - '.github/ISSUE_TEMPLATE/**'
+  - '.idea/**'
+  - 'deployment/**'
   - 'wiki/**'
 tests:
   - added|modified: '**/src/test/java/**/*.java'

.github/workflows/ci-go-functions.yaml

@@ -32,10 +32,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
-  changed_files_job:
+  preconditions:
     name: Preconditions
     runs-on: ubuntu-20.04
     outputs:
@@ -54,7 +54,11 @@ jobs:
       - name: Check changed files
         id: check_changes
         run: |
-          echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+          if [[ "${GITHUB_EVENT_NAME}" != "schedule" ]]; then
+            echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+          else
+            echo docs_only=false >> $GITHUB_OUTPUT
+          fi
 
       - name: Check if the PR has been approved for testing
         if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -65,8 +69,8 @@ jobs:
           build/pulsar_ci_tool.sh check_ready_to_test
 
   check-style:
-    needs: changed_files_job
-    if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+    needs: preconditions
+    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     name: Go ${{ matrix.go-version }} Functions style check
     runs-on: ubuntu-20.04
     strategy:

.github/workflows/ci-maven-cache-update.yaml

@@ -42,11 +42,13 @@ on:
     - cron: '30 */12 * * *'
 
 env:
-  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   update-maven-dependencies-cache:
     name: Update Maven dependency cache for ${{ matrix.name }}
+    env:
+      JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 45
 
@@ -75,6 +77,11 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Configure Gradle Enterprise
+        uses: ./.github/actions/gradle-enterprise
+        with:
+          token: ${{ secrets.GE_ACCESS_TOKEN }}
+
       - name: Detect changed files
         if: ${{ github.event_name != 'schedule' }}
         id:   changes
@@ -89,6 +96,7 @@ jobs:
         if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }}
         id: cache
         uses: actions/cache@v3
+        timeout-minutes: 5
         with:
           path: |
             ~/.m2/repository/*/*/*