diff --git a/.github/common/example/action.yml b/.github/common/example/action.yml new file mode 100644 index 000000000..a4babc58d --- /dev/null +++ b/.github/common/example/action.yml @@ -0,0 +1,40 @@ +name: Run examples +description: "Runs all examples on a single architecture and node version" + +inputs: + build-command: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Build + run: ${{ inputs.build-command }} + shell: bash + # Scylla docker setup copied from https://github.com/scylladb/scylla-rust-driver/blob/4532bdcf46955af57b902be25e0c419d9d6e3f20/.github/workflows/rust.yml + - name: Setup 3-node Scylla cluster + run: | + sudo sh -c "echo 2097152 >> /proc/sys/fs/aio-max-nr" + docker compose -f .github/docker-compose.yml up -d --wait + shell: bash + - name: Install examples dependencies + run: | + cd examples + npm i + shell: bash + - name: Run all examples + run: SCYLLA_URI=172.42.0.2:9042 npm run examples + shell: bash + - name: Stop the cluster + if: ${{ always() }} + run: docker compose -f .github/docker-compose.yml stop + shell: bash + - name: Print the cluster logs + if: ${{ always() }} + run: docker compose -f .github/docker-compose.yml logs + shell: bash + - name: Remove cluster + if: ${{ always() }} + run: docker compose -f .github/docker-compose.yml down + shell: bash diff --git a/.github/common/integration/action.yml b/.github/common/integration/action.yml new file mode 100644 index 000000000..45669a954 --- /dev/null +++ b/.github/common/integration/action.yml @@ -0,0 +1,33 @@ +name: Run integration tests +description: "Runs integration tests on a single architecture and node version" + +inputs: + build-command: + required: true + type: string + +runs: + using: "composite" + steps: + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + - name: Install ccm + # We need a copy of the repo for the ssl tests. + # When installing ccm as a package, we do not save the certificates + # that are present in the ./ssl directory + shell: bash + run: | + git clone https://github.com/scylladb/scylla-ccm.git + pip install --user --upgrade psutil + pip install --user ./scylla-ccm + - name: Build + run: ${{ inputs.build-command }} + shell: bash + - name: Run integration tests on scylla + env: + CCM_IS_SCYLLA: "true" + CCM_PATH: "./scylla-ccm" + shell: bash + run: npm run integration diff --git a/.github/setup/action.yml b/.github/common/setup/action.yml similarity index 89% rename from .github/setup/action.yml rename to .github/common/setup/action.yml index 2235b2e39..b6db498c4 100644 --- a/.github/setup/action.yml +++ b/.github/common/setup/action.yml @@ -1,4 +1,5 @@ name: Setup the driver +description: "Sets up all of the dependencies used in the CI" inputs: host: @@ -9,8 +10,8 @@ inputs: type: string node-version: required: false - type: number - default: 20 + type: string + default: "20" runs: diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1399725d2..1531feaa3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,7 +1,6 @@ name: Run benchmarks env: DEBUG: napi:* - APP_NAME: scylladb-javascript-driver PEDANTIC: true on: workflow_dispatch: @@ -19,7 +18,7 @@ jobs: name: Build and run benchmarks - benchmarks-only - node@${{ matrix.node }} runs-on: benchmarks-only steps: - - uses: ./.github/setup + - uses: ./.github/common/setup with: host: ${{ matrix.settings.host }} target: ${{ matrix.settings.target }} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 2c2b1db5f..12075889b 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,7 +1,6 @@ name: Check code quality env: DEBUG: napi:* - APP_NAME: scylladb-javascript-driver "on": push: branches: diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index c7009a82d..ea268b7bc 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -2,7 +2,6 @@ name: Run examples env: DEBUG: napi:* - APP_NAME: scylladb-javascript-driver PEDANTIC: true "on": push: @@ -19,7 +18,7 @@ jobs: host: - ubuntu-latest - ubuntu-24.04-arm - node: [ 20, 22, 24, current ] + node: [ 20, current ] include: - host: ubuntu-latest target: x86_64-unknown-linux-gnu @@ -29,37 +28,16 @@ jobs: build: npm run build -- --target aarch64-unknown-linux-gnu exclude: - host: ubuntu-24.04-arm - node: 22 - - host: ubuntu-24.04-arm - node: 24 + node: 20 name: Build and run examples - ${{ matrix.target }} - node@${{ matrix.node }} runs-on: ${{ matrix.host }} steps: - uses: actions/checkout@v4 - - uses: ./.github/setup + - uses: ./.github/common/setup with: host: ${{ matrix.host }} target: ${{ matrix.target }} node-version: ${{ matrix.node }} - - name: Build - run: ${{ matrix.build }} - shell: bash - # Scylla docker setup copied from https://github.com/scylladb/scylla-rust-driver/blob/main/.github/workflows/rust.yml - - name: Setup 3-node Scylla cluster - run: | - sudo sh -c "echo 2097152 >> /proc/sys/fs/aio-max-nr" - docker compose -f .github/docker-compose.yml up -d --wait - - name: Install examples dependencies - run: | - cd examples - npm i - - name: Run all examples - run: SCYLLA_URI=172.42.0.2:9042 npm run examples - - name: Stop the cluster - if: ${{ always() }} - run: docker compose -f .github/docker-compose.yml stop - - name: Print the cluster logs - if: ${{ always() }} - run: docker compose -f .github/docker-compose.yml logs - - name: Remove cluster - run: docker compose -f .github/docker-compose.yml down + - uses: ./.github/common/example + with: + build-command: ${{ matrix.build }} diff --git a/.github/workflows/extended-ci.yml b/.github/workflows/extended-ci.yml new file mode 100644 index 000000000..fc842e9fa --- /dev/null +++ b/.github/workflows/extended-ci.yml @@ -0,0 +1,66 @@ + +name: Extended CI +env: + DEBUG: napi:* + PEDANTIC: true + +on: + push: + # Currently we create tags before triggering a release. + # This setup ensures that we run this workflow before the release. + tags: + - "*" + workflow_dispatch: + +jobs: + # The goal of this workflow is to check if examples run without any errors + # This is a version that tests the full matrix of supported node versions and driver architectures + extended-examples-test: + strategy: + fail-fast: false + matrix: + host: + - ubuntu-latest + - ubuntu-24.04-arm + node: [ 20, 22, 24, current ] + include: + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + build: npm run build -- --target x86_64-unknown-linux-gnu + - host: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + build: npm run build -- --target aarch64-unknown-linux-gnu + name: Build and run examples - ${{ matrix.target }} - node@${{ matrix.node }} + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/common/setup + with: + host: ${{ matrix.host }} + target: ${{ matrix.target }} + node-version: ${{ matrix.node }} + - uses: ./.github/common/example + with: + build-command: ${{ matrix.build }} + extended-integration-tests: + strategy: + fail-fast: false + matrix: + settings: + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + build: npm run build -- --target x86_64-unknown-linux-gnu + node: [ 20, 22, 24, current ] + name: Build and run integration tests - ${{ matrix.settings.target }} - node@${{ matrix.node }} + runs-on: ${{ matrix.settings.host }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/common/setup + with: + host: ${{ matrix.settings.host }} + target: ${{ matrix.settings.target }} + node-version: ${{ matrix.node }} + - uses: ./.github/common/integration + with: + build-command: ${{ matrix.settings.build }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9e3da92ab..df0e92937 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -28,27 +28,11 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - uses: ./.github/setup + - uses: ./.github/common/setup with: host: ${{ matrix.settings.host }} target: ${{ matrix.settings.target }} - - uses: actions/setup-java@v4 + node-version: 20 + - uses: ./.github/common/integration with: - distribution: 'temurin' - java-version: '8' - - name: Install ccm - # We need a copy of the repo for the ssl tests. - # When installing ccm as a package, we do not save the certificates, - # that are present in the ./ssl directory - run: | - git clone https://github.com/scylladb/scylla-ccm.git - pip install --user --upgrade psutil - pip install --user https://github.com/scylladb/scylla-ccm/archive/master.zip - - name: Build - run: ${{ matrix.settings.build }} - shell: bash - - name: Run integration tests on scylla - env: - CCM_IS_SCYLLA: "true" - CCM_PATH: "./scylla-ccm" - run: npm run integration + build-command: ${{ matrix.settings.build }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56e2bbb08..80f4e4e7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,6 @@ name: Release package env: DEBUG: napi:* - APP_NAME: scylladb-javascript-driver PEDANTIC: true on: release: @@ -27,7 +26,7 @@ jobs: runs-on: ${{ matrix.settings.host }} steps: - uses: actions/checkout@v6 - - uses: ./.github/setup + - uses: ./.github/common/setup with: node-version: 24 host: ubuntu-latest diff --git a/.github/workflows/typescript-tests.yml b/.github/workflows/typescript-tests.yml index 688184a34..781cfe2f9 100644 --- a/.github/workflows/typescript-tests.yml +++ b/.github/workflows/typescript-tests.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/setup + - uses: ./.github/common/setup with: host: ubuntu-latest target: x86_64-unknown-linux-gnu diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index bb4397ca7..302f02f12 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/setup + - uses: ./.github/common/setup with: host: ubuntu-latest target: x86_64-unknown-linux-gnu diff --git a/MAINTENANCE.md b/MAINTENANCE.md index b2799b7d7..2b698f9b3 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -18,16 +18,84 @@ file to `.npmignore`. All files listed there will **NOT** be added to the npm pa When you remove a top level file or directory, remember to check (and remove if present) if the deleted file / directory was present in `.npmignore` file. +## CI + +We split the CI into two parts: + +- regular CI, that we run on each PR +- extended CI, that we run on each release + +We also have additional CI, that can be run occasionally: + +- benchmarks (can be triggered manually) +- documentation release (triggered on pushes to main) + +### Regular CI + +The regular CI consists of the following workflows: + +- Checking code quality (linters for JS and Rust) +- TypeScript tests +- Unit tests +- JSDoc linter +- Partial examples (see below) +- Partial integration tests + +### Extended CI + +The extended CI consists of the following workflow: + +- Full examples (see bellow) +- Full integration tests + +### Matrix + +We run examples on multiple node versions and architectures. + +- ✅ - means we run it both on regular and extended CI +- 🟠 - means we run it on extended CI only +- ❌ - means we do not run this configuration + +The motivation for such split is to reduce the execution time for CI that is run on each commit / PR, +while also ensuring the driver works correctly in most common configurations, when creating a new release. + +#### Examples + +Linux examples are run with ScyllaDB in a docker container. + +| | Linux x64 | Linux arm | MacOS Intel* | MacOS Arm* | +|-------------- |----------- |----------- |------------- |------------- | +| Node 20 | ✅ | 🟠 | ❌ (planned) | ❌ (planned) | +| Node 22 | 🟠 | 🟠 | ❌ (planned) | ❌ (planned) | +| Node 24 | 🟠 | 🟠 | ❌ (planned) | ❌ (planned) | +| Node current | ✅ | ✅ | ❌ (planned) | ❌ (planned) | + +*) Disabled due to problems with docker. There are plans to run them with Cassandra, +launched through CCM. Split between regular and extended CI is not yet decided. + +#### Integration tests + +| | Linux x64 | Linux arm**| MacOS Intel | MacOS Arm | +|-------------- |----------- |----------- |------------- |----------- | +| Node 20 | ✅ | ❌ | ❌ (planned) | ❌ | +| Node 22 | 🟠 | ❌ | ❌ (planned) | ❌ | +| Node 24 | 🟠 | ❌ | ❌ (planned) | ❌ | +| Node current | 🟠 | ❌ | ❌ (planned) | ❌ | + +**) Disabled due to problems with ccm + ## Releasing process 1. Bump the package version. Remember to update the version in `package-lock.json` in main directory, examples and benchmarks (see [example commit](https://github.com/scylladb/nodejs-rs-driver/pull/363/changes/41250609737052975129c7514439869324478008) on how to do that). -2. Create a release notes on GitHub. The version tag must match version from `package.json` with `v` prefix (for example: `v0.2.0`). +2. Create a new tag +3. Ensure the extended CI passes. +4. Create release notes on GitHub. The version tag must match version from `package.json` with `v` prefix (for example: `v0.2.0`). Once you publish release notes, CI action will trigger automatically. This action will build and publish the npm package. -3. Once the CI action finishes, check if it succeeded. If it failed, you will have to fix the underlying issue, and re-run the CI action. -4. Verify that the new release is visible at [npmjs site](https://www.npmjs.com/package/scylladb-driver-alpha). -5. Test the package, by installing it directly from npm. Go to `examples` directory, in `package.json` update the line: +5. Once the CI action finishes, check if it succeeded. If it failed, you will have to fix the underlying issue, and re-run the CI action. +6. Verify that the new release is visible at [npmjs site](https://www.npmjs.com/package/scylladb-driver-alpha). +7. Test the package, by installing it directly from npm. Go to `examples` directory, in `package.json` update the line: `"scylladb-driver-alpha": "file:./../"` to: `"scylladb-driver-alpha": ""`,