From 437ba2dbcb152833dddf1b05c331a32690abb28f Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 21 Dec 2023 10:20:51 +0100 Subject: [PATCH 01/14] Use Apache Logging Services reusable workflows This PR switches Flume to use the reusable workflows from apache/logging-parent. --- .github/dependabot.yaml | 40 +++++++++++ .github/workflows/build.yaml | 79 ++++++++++++++++++++++ .github/workflows/build.yml | 78 --------------------- .github/workflows/codeql-analysis.yaml | 41 +++++++++++ .github/workflows/codeql-analysis.yml | 70 ------------------- .github/workflows/merge-dependabot.yaml | 46 +++++++++++++ .github/workflows/scorecards-analysis.yaml | 40 +++++++++++ 7 files changed, 246 insertions(+), 148 deletions(-) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/codeql-analysis.yaml delete mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/merge-dependabot.yaml create mode 100644 .github/workflows/scorecards-analysis.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000000..1e15d7b8f0 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 2 +# Add Maven Central explicitly to work around: +# https://github.com/dependabot/dependabot-core/issues/8329 +registries: + maven-central: + type: maven-repository + url: https://repo.maven.apache.org/maven2 + +updates: + + - package-ecosystem: maven + directory: "/" + open-pull-requests-limit: 10 + schedule: + interval: "daily" + target-branch: "trunk" + registries: + - maven-central + +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: "daily" + target-branch: "trunk" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..fd51cee946 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,79 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: build + +on: + push: + branches: + - "trunk" + - "release/*" + paths-ignore: + - "**.adoc" + - "**.md" + - "**.txt" + pull_request: + paths-ignore: + - "**.adoc" + - "**.md" + - "**.txt" + +# If the branch is 'trunk' run once per commit. +# If the branch is 'release/*' allow only one concurrent run. +concurrency: + group: ${{ github.ref_name == 'trunk' && github.ref || github.ref_name }} + cancel-in-progress: true + +permissions: read-all + +jobs: + + build: + if: github.actor != 'dependabot[bot]' + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + with: + java-version: 8 + + deploy-snapshot: + needs: build + if: github.repository == 'apache/flume' && github.ref == 'refs/heads/trunk' + uses: apache/logging-parent/.github/workflows/deploy-snapshot-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + # Secrets for deployments + secrets: + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PW: ${{ secrets.NEXUS_PW }} + with: + java-version: 8 + + deploy-release: + needs: build + if: github.repository == 'apache/flume' && startsWith(github.ref_name, 'release/') + uses: apache/logging-parent/.github/workflows/deploy-release-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + # Secrets for deployments + secrets: + GPG_SECRET_KEY: ${{ secrets.LOGGING_GPG_SECRET_KEY }} + LOGGING_STAGE_DEPLOYER_USER: ${{ secrets.LOGGING_STAGE_DEPLOYER_USER }} + LOGGING_STAGE_DEPLOYER_PW: ${{ secrets.LOGGING_STAGE_DEPLOYER_PW }} + SVN_USERNAME: ${{ secrets.LOGGING_SVN_DEV_USERNAME }} + SVN_PASSWORD: ${{ secrets.LOGGING_SVN_DEV_PASSWORD }} + # Write permissions to allow the Maven `revision` property update, changelog release, etc. + permissions: + contents: write + with: + java-version: 8 + project-id: flume + site-enabled: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 686e1ed9f2..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache license, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the license for the specific language governing permissions and -# limitations under the license. - -name: build - -on: - push: - branches: - - trunk - - build-dev - pull_request: - -jobs: - build: - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - - steps: - - - name: Checkout repository - uses: actions/checkout@v2 - - # JDK 8 is needed for the build, and it is the primary bytecode target. - - name: Setup JDK 8 - uses: actions/setup-java@v2.3.0 - with: - distribution: temurin - java-version: 8 - java-package: jdk - architecture: x64 - cache: maven - - - name: Inspect environment (Linux) - if: runner.os == 'Linux' - run: env | grep '^JAVA' - - - name: Inspect environment (MacOS) - if: runner.os == 'macOS' - run: env | grep '^JAVA' - - - name: Show disk usage before - run: df -kh - - - name: Remove unwanted packages - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - - - name: Show disk usage before maven - run: df -kh - - - name: Build with Maven - timeout-minutes: 120 - shell: bash - run: | - ./mvnw clean verify -DredirectTestOutput=true \ - --show-version --batch-mode --errors --no-transfer-progress \ - -DtrimStackTrace=false \ - -Dsurefire.rerunFailingTestsCount=2 - - - name: Show disk usage after - run: df -kh diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000000..2e076ec586 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: codeql-analysis + +on: + push: + branches: [ 'trunk' ] + pull_request: + # The branches below must be a subset of the branches provided in `on.push.branches` + branches: [ 'trunk' ] + schedule: + - cron: '32 12 * * 5' + +permissions: read-all + +jobs: + + analyze: + uses: apache/logging-parent/.github/workflows/codeql-analysis-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + # Permissions required to publish Security Alerts + permissions: + actions: read + contents: read + security-events: write + with: + java-version: 8 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index b1490d04a9..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ trunk ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ trunk ] - schedule: - - cron: '15 0 * * 5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'java', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # 2.1.37 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # 2.1.37 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@959cbb7472c4d4ad70cdfe6f4976053fe48ab394 # 2.1.37 diff --git a/.github/workflows/merge-dependabot.yaml b/.github/workflows/merge-dependabot.yaml new file mode 100644 index 0000000000..b32d5060e6 --- /dev/null +++ b/.github/workflows/merge-dependabot.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: merge-dependabot + +on: + pull_request_target: + paths-ignore: + - "**.adoc" + - "**.md" + - "**.txt" + +permissions: read-all + +jobs: + + build: + if: github.repository == 'apache/flume' && github.event_name == 'pull_request_target' && github.actor == 'dependabot[bot]' + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + with: + java-version: 8 + + merge-dependabot: + needs: build + uses: apache/logging-parent/.github/workflows/merge-dependabot-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + permissions: + contents: write # to push changelog commits + pull-requests: write # to close the PR + secrets: + GPG_SECRET_KEY: ${{ secrets.LOGGING_GPG_SECRET_KEY }} # to sign commits + with: + java-version: 8 diff --git a/.github/workflows/scorecards-analysis.yaml b/.github/workflows/scorecards-analysis.yaml new file mode 100644 index 0000000000..3f51e17b15 --- /dev/null +++ b/.github/workflows/scorecards-analysis.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: scorecards-analysis + +on: + branch_protection_rule: + schedule: + - cron: "30 1 * * 6" + push: + branches: [ main ] + +permissions: read-all + +jobs: + + analysis: + uses: apache/logging-parent/.github/workflows/scorecards-analysis-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + permissions: + # Needed to upload the results to the code-scanning dashboard. + security-events: write + actions: read + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + with: + java-version: 8 From 33c14333c4afb259ccb90b3321365fafe33ac93e Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 21 Dec 2023 18:27:59 +0100 Subject: [PATCH 02/14] Bump ASF parent POM to version 31 --- flume-parent/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 4e4203e763..3a9c83c714 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache apache - 23 + 31 4.0.0 diff --git a/pom.xml b/pom.xml index 771e932252..4268ecebc6 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache apache - 23 + 31 4.0.0 From 3fbda41a83d1b82346c6534106e6a16de6052867 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 21 Dec 2023 21:47:02 +0100 Subject: [PATCH 03/14] Switch to `main` reusable builds --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fd51cee946..d88c0dbed2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -44,7 +44,7 @@ jobs: build: if: github.actor != 'dependabot[bot]' - uses: apache/logging-parent/.github/workflows/build-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@main with: java-version: 8 From bd424875b4af92cb96b4b1d73e8e16ea5c11aef5 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Fri, 22 Dec 2023 01:06:09 +0100 Subject: [PATCH 04/14] Disable `saveVersion.sh` script The script makes the build non-reproducible. --- flume-ng-core/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flume-ng-core/pom.xml b/flume-ng-core/pom.xml index 2ea97b7345..86661ddd1c 100644 --- a/flume-ng-core/pom.xml +++ b/flume-ng-core/pom.xml @@ -139,6 +139,8 @@ limitations under the License. + org.codehaus.mojo From ee4053452a570ca95ae1bc65fef1ae7e89902053 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 3 Oct 2024 17:41:52 +0200 Subject: [PATCH 05/14] Try to modernize the build --- .github/workflows/build.yaml | 33 +++++++++---------------- .github/workflows/merge-dependabot.yaml | 10 ++++++-- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d88c0dbed2..8e7d0a2ca9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,21 +22,7 @@ on: branches: - "trunk" - "release/*" - paths-ignore: - - "**.adoc" - - "**.md" - - "**.txt" pull_request: - paths-ignore: - - "**.adoc" - - "**.md" - - "**.txt" - -# If the branch is 'trunk' run once per commit. -# If the branch is 'release/*' allow only one concurrent run. -concurrency: - group: ${{ github.ref_name == 'trunk' && github.ref || github.ref_name }} - cancel-in-progress: true permissions: read-all @@ -44,30 +30,35 @@ jobs: build: if: github.actor != 'dependabot[bot]' - uses: apache/logging-parent/.github/workflows/build-reusable.yaml@main + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@rel/11.3.0 + secrets: + DV_ACCESS_TOKEN: ${{ startsWith(github.ref_name, 'release/') && '' || secrets.GE_ACCESS_TOKEN }} with: java-version: 8 + site-enabled: true + reproducibility-check-enabled: true + develocity-enabled: ${{ ! startsWith(github.ref_name, 'release/') }} deploy-snapshot: needs: build if: github.repository == 'apache/flume' && github.ref == 'refs/heads/trunk' - uses: apache/logging-parent/.github/workflows/deploy-snapshot-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + uses: apache/logging-parent/.github/workflows/deploy-snapshot-reusable.yaml@rel/11.3.0 # Secrets for deployments secrets: - NEXUS_USER: ${{ secrets.NEXUS_USER }} - NEXUS_PW: ${{ secrets.NEXUS_PW }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }} with: java-version: 8 deploy-release: needs: build if: github.repository == 'apache/flume' && startsWith(github.ref_name, 'release/') - uses: apache/logging-parent/.github/workflows/deploy-release-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + uses: apache/logging-parent/.github/workflows/deploy-release-reusable.yaml@rel/11.3.0 # Secrets for deployments secrets: GPG_SECRET_KEY: ${{ secrets.LOGGING_GPG_SECRET_KEY }} - LOGGING_STAGE_DEPLOYER_USER: ${{ secrets.LOGGING_STAGE_DEPLOYER_USER }} - LOGGING_STAGE_DEPLOYER_PW: ${{ secrets.LOGGING_STAGE_DEPLOYER_PW }} + NEXUS_USERNAME: ${{ secrets.LOGGING_STAGE_DEPLOYER_USER }} + NEXUS_PASSWORD: ${{ secrets.LOGGING_STAGE_DEPLOYER_PW }} SVN_USERNAME: ${{ secrets.LOGGING_SVN_DEV_USERNAME }} SVN_PASSWORD: ${{ secrets.LOGGING_SVN_DEV_PASSWORD }} # Write permissions to allow the Maven `revision` property update, changelog release, etc. diff --git a/.github/workflows/merge-dependabot.yaml b/.github/workflows/merge-dependabot.yaml index b32d5060e6..cf8bb2b037 100644 --- a/.github/workflows/merge-dependabot.yaml +++ b/.github/workflows/merge-dependabot.yaml @@ -30,13 +30,19 @@ jobs: build: if: github.repository == 'apache/flume' && github.event_name == 'pull_request_target' && github.actor == 'dependabot[bot]' - uses: apache/logging-parent/.github/workflows/build-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@rel/11.3.0 + secrets: + DV_ACCESS_TOKEN: ${{ secrets.GE_ACCESS_TOKEN }} with: java-version: 8 + develocity-enabled: true + reproducibility-check-enabled: false merge-dependabot: needs: build - uses: apache/logging-parent/.github/workflows/merge-dependabot-reusable.yaml@e45457c683302242be5e8e7c3c33edf8f0e0ec0e # 10.4.0 + uses: apache/logging-parent/.github/workflows/merge-dependabot-reusable.yaml@rel/11.3.0 + with: + java-version: 8 permissions: contents: write # to push changelog commits pull-requests: write # to close the PR From d31231f5eb159661c073b1af2a7bd750943caba5 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 3 Oct 2024 17:43:40 +0200 Subject: [PATCH 06/14] Remove deprecated `site-enabled` parameter --- .github/workflows/build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e7d0a2ca9..c37482e457 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,4 +67,3 @@ jobs: with: java-version: 8 project-id: flume - site-enabled: true From c601c4faaa4943cab0d53a3036fc99a5b96f6e54 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 3 Oct 2024 17:45:59 +0200 Subject: [PATCH 07/14] Fix `merge-dependabot` workflow --- .github/workflows/merge-dependabot.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/merge-dependabot.yaml b/.github/workflows/merge-dependabot.yaml index cf8bb2b037..dc0b4adfc0 100644 --- a/.github/workflows/merge-dependabot.yaml +++ b/.github/workflows/merge-dependabot.yaml @@ -48,5 +48,3 @@ jobs: pull-requests: write # to close the PR secrets: GPG_SECRET_KEY: ${{ secrets.LOGGING_GPG_SECRET_KEY }} # to sign commits - with: - java-version: 8 From 3182049c47a14dea99fa49489c02a1a582bef84c Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Fri, 4 Oct 2024 19:51:54 +0200 Subject: [PATCH 08/14] Remove external artifacts from distribution --- flume-ng-dist/pom.xml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml index 373b499fea..19d5b9ed6f 100644 --- a/flume-ng-dist/pom.xml +++ b/flume-ng-dist/pom.xml @@ -132,18 +132,10 @@ org.apache.flume flume-kafka-channel - - org.apache.flume - flume-hdfs-sink - org.apache.flume.flume-ng-sinks flume-irc-sink - - org.apache.flume - flume-hbase2-sink - org.apache.flume.flume-ng-sinks flume-http-sink @@ -156,10 +148,6 @@ org.apache.flume flume-kafka-sink - - org.apache.flume - flume-hive-sink - org.apache.flume.flume-ng-sources flume-scribe-source @@ -192,10 +180,6 @@ org.apache.flume flume-ng-environment-variable-config-filter - - org.apache.flume - flume-hadoop-credential-store-config-filter - org.apache.flume flume-ng-external-process-config-filter From d3e41aea7d9a24e54a7611d1b489072429b99de3 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Fri, 4 Oct 2024 22:39:25 +0200 Subject: [PATCH 09/14] Make `VersionInfo` reproducible --- flume-ng-core/pom.xml | 58 +++------------- flume-ng-core/scripts/saveVersion.ps1 | 61 ---------------- flume-ng-core/scripts/saveVersion.sh | 69 ------------------- .../org/apache/flume/tools/VersionInfo.java | 59 ++++++++-------- .../apache/flume/tools/VersionInfo.properties | 25 +++++++ flume-parent/pom.xml | 10 ++- 6 files changed, 73 insertions(+), 209 deletions(-) delete mode 100644 flume-ng-core/scripts/saveVersion.ps1 delete mode 100755 flume-ng-core/scripts/saveVersion.sh create mode 100644 flume-ng-core/src/main/resources/org/apache/flume/tools/VersionInfo.properties diff --git a/flume-ng-core/pom.xml b/flume-ng-core/pom.xml index 86661ddd1c..d4107ff14d 100644 --- a/flume-ng-core/pom.xml +++ b/flume-ng-core/pom.xml @@ -38,6 +38,15 @@ limitations under the License. + + + + + src/main/resources + true + + + @@ -139,32 +148,6 @@ limitations under the License. - org.codehaus.mojo @@ -199,29 +182,6 @@ limitations under the License. - - org.apache.maven.plugins - maven-antrun-plugin - ${mvn-antrun-plugin.version} - - - generate-version - generate-sources - - - - - - - - - - run - - - - org.codehaus.mojo diff --git a/flume-ng-core/scripts/saveVersion.ps1 b/flume-ng-core/scripts/saveVersion.ps1 deleted file mode 100644 index dff88134ee..0000000000 --- a/flume-ng-core/scripts/saveVersion.ps1 +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to generate the annotation of package info that -# records the version, revision, branch, url, user and timestamp. - -$version=$args[0] -$buildDirectory=$args[1] -$outputFile= "$buildDirectory\generated-sources\java\org\apache\flume\package-info.java" -$user = $Env:username -$date = Get-Date - -cmd /c svn info 2>&1 | Out-Null -if ( $LastExitCode -eq 0 ) { - $revision=svn info | % { if( $_ -match "Last Changed Rev: (?.*)" ) { $matches['rev'];} } - $url=svn info | % { if( $_ -match "URL: (?.*)" ) { $matches['url'];} } - $branch= if( $url -match ".*(?(branches.*)|(tags.*)|(trunk.*))" ) { $matches['branch']; } else { "Unknown"; } -} -else { - cmd /c git rev-parse HEAD 2>&1 | Out-Null - if ( $LastExitCode -eq 0 ) { - $revision=$(git log -1 --pretty=format:"%H") - $branch=$(git name-rev --name-only HEAD) - $remote=$(git config branch.$branch.remote) - $url=$(git config remote.$remote.url) - } - else { - revision="Unknown" - branch="Unknown" - url="file://$cwd" - } -} - -$srcChecksum="N/A" - - -$fileContent = @" -/* - * Generated by scripts/saveVersion.ps1 - */ -@VersionAnnotation(version="$version", revision="$revision", branch="$branch", - user="$user", date="$date", url="$url", - srcChecksum="$srcChecksum") -package org.apache.flume; -"@ - -New-Item $outputFile -value $fileContent -force -type file - diff --git a/flume-ng-core/scripts/saveVersion.sh b/flume-ng-core/scripts/saveVersion.sh deleted file mode 100755 index ad3f8b195d..0000000000 --- a/flume-ng-core/scripts/saveVersion.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to generate the annotation of package info that -# records the version, revision, branch, url, user and timestamp. - -unset LANG -unset LC_CTYPE -unset LC_TIME -version=$1 -buildDirectory=$2 -user=`whoami` -date=`date` -dir=`pwd` -cwd=`dirname $dir` -if [ -d ../.svn ]; then - revision=`svn info ../ | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` - url=`svn info ../ | sed -n -e 's/URL: \(.*\)/\1/p'` - branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \ - -e 's,.*\(tags/.*\)$,\1,p' \ - -e 's,.*trunk$,trunk,p'` -elif git rev-parse HEAD 2>/dev/null > /dev/null ; then - revision=`git log -1 --pretty=format:"%H"` - hostname=`hostname` - branch=`git branch | sed -n -e 's/^* //p'` - url="git://${hostname}${cwd}" -else - revision="Unknown" - branch="Unknown" - url="file://$cwd" -fi - -if [ -n "$(which md5sum)" ]; then - srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | \ - xargs md5sum | md5sum | cut -d ' ' -f 1` -else - srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | \ - xargs md5 | md5 | cut -d ' ' -f 1` -fi - -mkdir -p $buildDirectory/generated-sources/java/org/apache/flume/ -cat << EOF | \ - sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \ - -e "s|URL|$url|" -e "s/REV/$revision/" \ - -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \ - > $buildDirectory/generated-sources/java/org/apache/flume/package-info.java -/* - * Generated by scripts/saveVersion.sh - */ -@VersionAnnotation(version="VERSION", revision="REV", branch="BRANCH", - user="USER", date="DATE", url="URL", - srcChecksum="SRCCHECKSUM") -package org.apache.flume; -EOF \ No newline at end of file diff --git a/flume-ng-core/src/main/java/org/apache/flume/tools/VersionInfo.java b/flume-ng-core/src/main/java/org/apache/flume/tools/VersionInfo.java index 574e527ce6..ca98c4b0d9 100644 --- a/flume-ng-core/src/main/java/org/apache/flume/tools/VersionInfo.java +++ b/flume-ng-core/src/main/java/org/apache/flume/tools/VersionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,12 @@ */ package org.apache.flume.tools; -import org.apache.flume.VersionAnnotation; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Properties; /* * This class provides version info of Flume NG @@ -25,72 +30,70 @@ public class VersionInfo { - private static Package myPackage; - private static VersionAnnotation version; + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss z yyyy"); + private static final Properties properties = new Properties(); static { - myPackage = VersionAnnotation.class.getPackage(); - version = myPackage.getAnnotation(VersionAnnotation.class); - } - - /** - * Get the meta-data for the Flume package. - * @return - */ - static Package getPackage() { - return myPackage; + try (InputStream is = VersionInfo.class.getResourceAsStream("VersionInfo.properties");) { + properties.load(is); + } catch (IOException e) { + throw new IllegalStateException(e); + } } /** * Get the Flume version. + * * @return the Flume version string, eg. "1.1" */ public static String getVersion() { - return version != null ? version.version() : "Unknown"; + return properties.getProperty("version", "Unknown"); } /** * Get the subversion revision number for the root directory + * * @return the revision number, eg. "100755" */ public static String getRevision() { - if (version != null - && version.revision() != null - && !version.revision().isEmpty()) { - return version.revision(); - } - return "Unknown"; + return properties.getProperty("revision", "Unknown"); } /** * Get the branch on which this originated. + * * @return The branch name, e.g. "trunk" or "branches/branch-1.1" */ public static String getBranch() { - return version != null ? version.branch() : "Unknown"; + return properties.getProperty("branch", "Unknown"); } /** * The date that Flume was compiled. + * * @return the compilation date in unix date format */ public static String getDate() { - return version != null ? version.date() : "Unknown"; + String timestamp = properties.getProperty("date"); + return timestamp == null + ? "Unknown" + : Instant.parse(timestamp).atZone(ZoneOffset.UTC).format(formatter); } /** * The user that compiled Flume. + * * @return the username of the user */ public static String getUser() { - return version != null ? version.user() : "Unknown"; + return properties.getProperty("user", "Unknown"); } /** * Get the subversion URL for the root Flume directory. */ public static String getUrl() { - return version != null ? version.url() : "Unknown"; + return properties.getProperty("url", "Unknown"); } /** @@ -98,7 +101,7 @@ public static String getUrl() { * built. **/ public static String getSrcChecksum() { - return version != null ? version.srcChecksum() : "Unknown"; + return properties.getProperty("srcChecksum", "Unknown"); } /** diff --git a/flume-ng-core/src/main/resources/org/apache/flume/tools/VersionInfo.properties b/flume-ng-core/src/main/resources/org/apache/flume/tools/VersionInfo.properties new file mode 100644 index 0000000000..59ee36cd96 --- /dev/null +++ b/flume-ng-core/src/main/resources/org/apache/flume/tools/VersionInfo.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +## +# Contains the version information +version = ${project.version} +revision = N/A +branch = trunk +date = ${maven.build.timestamp} +user = N/A +url = https://github.com/apache/logging-flume +srcChecksum = N/A \ No newline at end of file diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 3a9c83c714..92cbbf9ef1 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -33,8 +33,14 @@ limitations under the License. Apache Flume - - 2022-01-02T00:00:00Z + + 2024-09-24T13:33:09Z UTF-8 From 1234ca8765bce5fb61a1d1c1b61fece5b1f4ca98 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Fri, 4 Oct 2024 23:14:10 +0200 Subject: [PATCH 10/14] Add Develocity support --- .mvn/develocity.xml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .mvn/develocity.xml diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml new file mode 100644 index 0000000000..b4ca7ef406 --- /dev/null +++ b/.mvn/develocity.xml @@ -0,0 +1,30 @@ + + + logging-flume + + https://ge.apache.org + + + + true + true + + + 0.0.0.0 + + + + + + + false + + + + false + + + false + + + From d1c2489f1b02177084380033269f7c9ad391665c Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Sat, 5 Oct 2024 00:14:51 +0200 Subject: [PATCH 11/14] Add Develocity versions --- flume-parent/pom.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 92cbbf9ef1..0c1b811a52 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -70,6 +70,8 @@ limitations under the License. 1.10.0 5.1.0 10.14.2.0 + 1.22.2 + 2.0.1 4.1.18 2.13.2 2.13.2.1 @@ -461,6 +463,14 @@ limitations under the License. + + + + com.gradle + develocity-maven-extension + ${develocity-maven-plugin.version} + + org.apache.maven.plugins maven-antrun-plugin @@ -817,6 +827,13 @@ limitations under the License. ${commons-text.version} + + + com.gradle + common-custom-user-data-maven-extension + ${develocity-user-data-extension.version} + + com.google.guava guava From 944b2f735b1bd261cecb4fdc4a89fbcde1227d5d Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Sat, 5 Oct 2024 00:20:15 +0200 Subject: [PATCH 12/14] Move Develocity config to main `pom.xml` --- flume-parent/pom.xml | 16 ---------------- pom.xml | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 0c1b811a52..86431a3f9a 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -70,8 +70,6 @@ limitations under the License. 1.10.0 5.1.0 10.14.2.0 - 1.22.2 - 2.0.1 4.1.18 2.13.2 2.13.2.1 @@ -464,13 +462,6 @@ limitations under the License. - - - com.gradle - develocity-maven-extension - ${develocity-maven-plugin.version} - - org.apache.maven.plugins maven-antrun-plugin @@ -827,13 +818,6 @@ limitations under the License. ${commons-text.version} - - - com.gradle - common-custom-user-data-maven-extension - ${develocity-user-data-extension.version} - - com.google.guava guava diff --git a/pom.xml b/pom.xml index 4268ecebc6..bfaa079014 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,12 @@ limitations under the License. Apache Flume Project + + + 1.22.2 + 2.0.1 + + flume-bom flume-parent @@ -52,4 +58,24 @@ limitations under the License. build-support + + + + + + com.gradle + develocity-maven-extension + ${develocity-maven-plugin.version} + + + + + com.gradle + common-custom-user-data-maven-extension + ${develocity-user-data-extension.version} + + + + + From f33a7646e21c6ef6810f3f20f1c0a8c5156d8b37 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Sat, 5 Oct 2024 00:22:17 +0200 Subject: [PATCH 13/14] Fix POM file --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index bfaa079014..d4090990d5 100644 --- a/pom.xml +++ b/pom.xml @@ -62,11 +62,11 @@ limitations under the License. - + com.gradle develocity-maven-extension - ${develocity-maven-plugin.version} - + ${develocity-maven-dependency.version} + From 5bf1149bead56a0beaa63761cf49470e0b0ff1c6 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Sat, 5 Oct 2024 00:27:03 +0200 Subject: [PATCH 14/14] Bump used JDK version to 11 --- .github/workflows/build.yaml | 6 +++--- .github/workflows/codeql-analysis.yaml | 2 +- .github/workflows/merge-dependabot.yaml | 4 ++-- .github/workflows/scorecards-analysis.yaml | 2 +- flume-parent/pom.xml | 6 ++---- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c37482e457..06a3b9e734 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,7 +34,7 @@ jobs: secrets: DV_ACCESS_TOKEN: ${{ startsWith(github.ref_name, 'release/') && '' || secrets.GE_ACCESS_TOKEN }} with: - java-version: 8 + java-version: 11 site-enabled: true reproducibility-check-enabled: true develocity-enabled: ${{ ! startsWith(github.ref_name, 'release/') }} @@ -48,7 +48,7 @@ jobs: NEXUS_USERNAME: ${{ secrets.NEXUS_USER }} NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }} with: - java-version: 8 + java-version: 11 deploy-release: needs: build @@ -65,5 +65,5 @@ jobs: permissions: contents: write with: - java-version: 8 + java-version: 11 project-id: flume diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index 2e076ec586..03b612f37f 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -38,4 +38,4 @@ jobs: contents: read security-events: write with: - java-version: 8 + java-version: 11 diff --git a/.github/workflows/merge-dependabot.yaml b/.github/workflows/merge-dependabot.yaml index dc0b4adfc0..ecb73c1731 100644 --- a/.github/workflows/merge-dependabot.yaml +++ b/.github/workflows/merge-dependabot.yaml @@ -34,7 +34,7 @@ jobs: secrets: DV_ACCESS_TOKEN: ${{ secrets.GE_ACCESS_TOKEN }} with: - java-version: 8 + java-version: 11 develocity-enabled: true reproducibility-check-enabled: false @@ -42,7 +42,7 @@ jobs: needs: build uses: apache/logging-parent/.github/workflows/merge-dependabot-reusable.yaml@rel/11.3.0 with: - java-version: 8 + java-version: 11 permissions: contents: write # to push changelog commits pull-requests: write # to close the PR diff --git a/.github/workflows/scorecards-analysis.yaml b/.github/workflows/scorecards-analysis.yaml index 3f51e17b15..6371a0f46a 100644 --- a/.github/workflows/scorecards-analysis.yaml +++ b/.github/workflows/scorecards-analysis.yaml @@ -37,4 +37,4 @@ jobs: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout with: - java-version: 8 + java-version: 11 diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 86431a3f9a..7c4ce31dbb 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -45,8 +45,7 @@ limitations under the License. UTF-8 - 1.8 - 1.8 + 8 rgoers@apache.org @@ -485,8 +484,7 @@ limitations under the License. maven-compiler-plugin ${mvn-compiler-plugin.version} - ${sourceJavaVersion} - ${targetJavaVersion} + ${releaseJavaVersion}