diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a238f2c9999..6db73a98385 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,16 @@ .merge_train_rule: &merge_train_rule - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' + INTEGRATION_TEST: "no" INTEGRATION_TEST_SCOPE: mr - FUNCTIONAL_TEST: 'yes' + FUNCTIONAL_TEST: "yes" FUNCTIONAL_TEST_SCOPE: mr-slim FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" workflow: rules: @@ -32,33 +32,36 @@ workflow: # For manual pipelines - if: $CI_PIPELINE_SOURCE == "web" + # For trigger pipelines + - if: $CI_PIPELINE_SOURCE == "trigger" + # For push to main - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH =~ /^core_/) variables: - UNIT_TEST: 'no' - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'yes' + UNIT_TEST: "no" + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "yes" FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 5 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" FUNCTIONAL_TEST_TIME_LIMIT: 3600 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" auto_cancel: on_new_commit: interruptible # For merge-trains that need to be fast-tracked - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' && $CI_MERGE_REQUEST_LABELS =~ /fast-track/ variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'no' - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "no" + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" # For normal merge-trains - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' @@ -67,75 +70,75 @@ workflow: # For MRs with integration suite - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run tests/ variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'yes' + INTEGRATION_TEST: "yes" INTEGRATION_TEST_SCOPE: mr - FUNCTIONAL_TEST: 'no' + FUNCTIONAL_TEST: "no" FUNCTIONAL_TEST_SCOPE: mr-slim FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" # For MRs with nightly - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run nightly/ variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'yes' + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "yes" FUNCTIONAL_TEST_SCOPE: nightly FUNCTIONAL_TEST_REPEAT: 5 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" # For MRs with weekly - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run weekly/ variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'yes' + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "yes" FUNCTIONAL_TEST_SCOPE: weekly FUNCTIONAL_TEST_REPEAT: 1 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" FUNCTIONAL_TEST_TIME_LIMIT: 9000 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" # For MRs with heavy suite - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run functional tests/ variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'yes' + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "yes" FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: '' - CLUSTER_H100: '' - PUBLISH: 'no' + CLUSTER_A100: "" + CLUSTER_H100: "" + PUBLISH: "no" # Default MRs - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' variables: - UNIT_TEST: 'yes' + UNIT_TEST: "yes" UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: 'no' - FUNCTIONAL_TEST: 'no' - PUBLISH: 'no' + INTEGRATION_TEST: "no" + FUNCTIONAL_TEST: "no" + PUBLISH: "no" - when: never @@ -157,109 +160,109 @@ default: variables: BUILD: - value: 'yes' + value: "yes" UNIT_TEST: - value: 'yes' + value: "yes" options: - - 'yes' - - 'no' + - "yes" + - "no" description: To run the funtional test suite UNIT_TEST_REPEAT: - value: '1' - description: 'Number of repetitions' + value: "1" + description: "Number of repetitions" UNIT_TEST_TIMEOUT: - value: '30' + value: "30" description: Timeout (minutes) for Unit tests (all repeats) INTEGRATION_TEST: - value: 'yes' + value: "yes" options: - - 'yes' - - 'no' + - "yes" + - "no" description: To run the integration test suite INTEGRATION_TEST_SCOPE: - value: 'mr' + value: "mr" options: - - 'mr' - - 'nightly' - - 'weekly' - - 'pre-release' - - 'release' - description: 'Testsuite to run (only for INTEGRATION_TEST=yes)' + - "mr" + - "nightly" + - "weekly" + - "pre-release" + - "release" + description: "Testsuite to run (only for INTEGRATION_TEST=yes)" INTEGRATION_TEST_TIME_LIMIT: - value: '900' - description: 'Timeout in seconds per test' + value: "900" + description: "Timeout in seconds per test" INTEGRATION_TEST_CASES: - value: 'all' + value: "all" description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite." FUNCTIONAL_TEST: - value: 'yes' + value: "yes" options: - - 'yes' - - 'no' + - "yes" + - "no" description: To run the funtional test suite FUNCTIONAL_TEST_SCOPE: - value: 'mr' + value: "mr" options: - - 'mr' - - 'nightly' - - 'weekly' - - 'pre-release' - - 'release' - description: 'Testsuite to run (only for FUNCTIONAL_TEST=yes)' + - "mr" + - "nightly" + - "weekly" + - "pre-release" + - "release" + description: "Testsuite to run (only for FUNCTIONAL_TEST=yes)" FUNCTIONAL_TEST_REPEAT: - value: '5' - description: 'Number of repetitions per test' + value: "5" + description: "Number of repetitions per test" FUNCTIONAL_TEST_TIME_LIMIT: - value: '2700' - description: 'Timeout in seconds per test' + value: "2700" + description: "Timeout in seconds per test" FUNCTIONAL_TEST_CASES: - value: 'all' + value: "all" description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite." FUNCTIONAL_TEST_NAME: - description: 'Name of functional test run (only for pre-release and release)' - value: '$$CI_COMMIT_SHA' + description: "Name of functional test run (only for pre-release and release)" + value: "$$CI_COMMIT_SHA" FUNCTIONAL_TEST_RECORD_CHECKPOINTS: - value: 'no' - description: 'Record golden checkpoints' + value: "no" + description: "Record golden checkpoints" options: - - 'yes' - - 'no' + - "yes" + - "no" CLUSTER_A100: - value: 'dgxa100_dracooci' + value: "dgxa100_dracooci" options: - - 'dgxa100_dracooci' - - 'dgxa100_dracooci-ord' - description: 'Cluster for A100 workloads' + - "dgxa100_dracooci" + - "dgxa100_dracooci-ord" + description: "Cluster for A100 workloads" CLUSTER_H100: - value: 'dgxh100_coreweave' + value: "dgxh100_coreweave" options: - - 'dgxh100_coreweave' - - 'dgxh100_eos' - description: 'Cluster for H100 workloads' + - "dgxh100_coreweave" + - "dgxh100_eos" + description: "Cluster for H100 workloads" CLUSTER_GB200: - value: 'dgxgb200_oci-hsg' + value: "dgxgb200_oci-hsg" options: - - 'dgxgb200_oci-hsg' - description: 'Cluster for H100 workloads' + - "dgxgb200_oci-hsg" + description: "Cluster for H100 workloads" PUBLISH: - value: 'no' + value: "no" options: - - 'yes' - - 'no' + - "yes" + - "no" description: Build and publish a wheel to PyPi PUBLISH_COMMIT: - value: '$$CI_COMMIT_SHA' + value: "$$CI_COMMIT_SHA" description: Which commit to publish PUBLISH_VERSION_BUMP_BRANCH: - value: '$$CI_COMMIT_BRANCH' + value: "$$CI_COMMIT_BRANCH" description: Which branch to target for version bump PUBLISH_SCOPE: - value: 'code-freeze' + value: "code-freeze" options: - - 'code-freeze' - - 'release' - - 'review-reminder' - - 'upgrade-dependencies' + - "code-freeze" + - "release" + - "review-reminder" + - "upgrade-dependencies" description: Type of publish (freeze or final release) # CI wide variables @@ -267,7 +270,7 @@ variables: CI_MCORE_DEV_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci_dev CI_NEMO_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/nemo_ci UTILITY_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_utility - TE_GIT_REF: '' + TE_GIT_REF: "" include: - .gitlab/stages/00.pre.yml diff --git a/tools/trigger_internal_ci.md b/tools/trigger_internal_ci.md new file mode 100644 index 00000000000..313ed98a437 --- /dev/null +++ b/tools/trigger_internal_ci.md @@ -0,0 +1,73 @@ +# trigger_internal_ci.py + +Pushes the current branch to the internal GitLab remote and triggers a CI +pipeline — without touching the GitLab UI. + +## Prerequisites + +**1. Add the internal GitLab as a git remote** (skip if you already have one configured): + +```bash +git remote add gitlab git@:/.git +``` + +To check existing remotes: `git remote -v` + +**2. Obtain a pipeline trigger token:** + +1. Open the internal GitLab project in your browser. +2. Go to **Settings → CI/CD → Pipeline trigger tokens**. +3. Click **Add new token**, give it a description, and click **Create**. +4. Copy the generated token (starts with `glptt-`). +5. Store it in your environment to avoid passing it on every invocation: + +Reach out to @mcore-ci in case you don't have access to the settings page. + +```bash +export GITLAB_TRIGGER_TOKEN=glptt- +``` + +## Usage + +```bash +python tools/trigger_internal_ci.py \ + --gitlab-origin gitlab \ + [--trigger-token glptt-] \ + [--functional-test-scope mr] \ + [--functional-test-repeat 5] \ + [--functional-test-cases all] \ + [--dry-run] +``` + +| Argument | Default | Description | +|---|---|---| +| `--gitlab-origin` | *(required)* | Git remote name for the internal GitLab | +| `--trigger-token` | `$GITLAB_TRIGGER_TOKEN` | Pipeline trigger token | +| `--functional-test-scope` | `mr` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | +| `--functional-test-repeat` | `5` | `FUNCTIONAL_TEST_REPEAT` pipeline variable | +| `--functional-test-cases` | `all` | `FUNCTIONAL_TEST_CASES` pipeline variable | +| `--dry-run` | off | Print what would happen without pushing or triggering | + +## Example + +```bash +# Dry run — no push, no trigger +python tools/trigger_internal_ci.py --gitlab-origin gitlab --dry-run + +# Real run — uses token from environment +python tools/trigger_internal_ci.py --gitlab-origin gitlab +``` + +## Expected behavior + +``` +Current branch: my-feature-branch +Everything up-to-date +Triggering pipeline on https:// project 19378 @ pull-request/my-feature-branch +Pipeline triggered: https://///-/pipelines/123456 +``` + +1. The current branch is detected from git. +2. The branch is force-pushed to the GitLab remote as `pull-request/`. +3. A pipeline is triggered on that ref with the configured test variables. +4. The URL of the newly created pipeline is printed. diff --git a/tools/trigger_internal_ci.py b/tools/trigger_internal_ci.py new file mode 100644 index 00000000000..3b97c92332e --- /dev/null +++ b/tools/trigger_internal_ci.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI tool to trigger the internal GitLab CI pipeline from a local branch. + +Pushes the current branch to the internal GitLab remote under the +pull-request/ naming convention and triggers a pipeline with +the specified test configuration. +""" + +import argparse +import logging +import os +import subprocess +import sys +from urllib.parse import urlparse + +import gitlab # python-gitlab + +GITLAB_PROJECT_ID = 19378 +GITLAB_BRANCH_PREFIX = "pull-request" + +PIPELINE_VARIABLES_FIXED = { + "UNIT_TEST": "no", + "INTEGRATION_TEST": "no", +} + +logger = logging.getLogger(__name__) + + +def get_remote_url(origin): + """Return the fetch URL configured for the given git remote name.""" + result = subprocess.run( + ["git", "remote", "get-url", origin], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def get_gitlab_hostname(remote_url): + """Extract the hostname (without port) from an SSH or HTTPS remote URL.""" + if remote_url.startswith("git@"): + hostname = remote_url.split("@", 1)[1].split(":")[0] + else: + hostname = urlparse(remote_url).hostname + return hostname.split(":")[0] + + +def get_current_branch(): + """Return the name of the currently checked-out git branch.""" + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def git_push(origin, target_branch, dry_run=False): + """Force-push HEAD to the given branch on the named git remote.""" + if dry_run: + logger.info("[DRY RUN] Would push HEAD to remote '%s' as %s", origin, target_branch) + return + subprocess.run( + ["git", "push", origin, f"HEAD:{target_branch}", "--force"], + check=True, + ) + + +def trigger_pipeline(gitlab_url, trigger_token, ref, pipeline_vars, dry_run=False): + """Trigger a GitLab pipeline on the given ref with the provided variables.""" + if dry_run: + logger.info( + "[DRY RUN] Would trigger pipeline on https://%s project %s @ %s", + gitlab_url, + GITLAB_PROJECT_ID, + ref, + ) + return + logger.info( + "Triggering pipeline on https://%s project %s @ %s", gitlab_url, GITLAB_PROJECT_ID, ref + ) + gl = gitlab.Gitlab(f"https://{gitlab_url}") + project = gl.projects.get(GITLAB_PROJECT_ID, lazy=True) + pipeline = project.trigger_pipeline(ref=ref, token=trigger_token, variables=pipeline_vars) + logger.info("Pipeline triggered: %s", pipeline.web_url) + + +def main(): + """Parse arguments and orchestrate the push and pipeline trigger flow.""" + parser = argparse.ArgumentParser( + description="Trigger the internal GitLab CI pipeline for the current branch." + ) + parser.add_argument( + "--gitlab-origin", + required=True, + help="Name of the git remote pointing to the internal GitLab (e.g. gitlab)", + ) + parser.add_argument( + "--trigger-token", + default=os.environ.get("GITLAB_TRIGGER_TOKEN"), + help="GitLab pipeline trigger token (or set GITLAB_TRIGGER_TOKEN env var)", + ) + parser.add_argument( + "--functional-test-scope", + default="mr", + help="FUNCTIONAL_TEST_SCOPE pipeline variable (default: mr)", + ) + parser.add_argument( + "--functional-test-repeat", + type=int, + default=5, + help="FUNCTIONAL_TEST_REPEAT pipeline variable (default: 5)", + ) + parser.add_argument( + "--functional-test-cases", + default="all", + help="FUNCTIONAL_TEST_CASES pipeline variable (default: all)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print actions without executing git push or pipeline trigger", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(message)s") + + if not args.trigger_token: + logger.error("--trigger-token or GITLAB_TRIGGER_TOKEN not set") + sys.exit(1) + + branch = get_current_branch() + logger.info("Current branch: %s", branch) + + remote_url = get_remote_url(args.gitlab_origin) + gitlab_hostname = get_gitlab_hostname(remote_url) + + target_branch = f"{GITLAB_BRANCH_PREFIX}/{branch}" + + git_push(args.gitlab_origin, target_branch, dry_run=args.dry_run) + + pipeline_vars = { + **PIPELINE_VARIABLES_FIXED, + "FUNCTIONAL_TEST_SCOPE": args.functional_test_scope, + "FUNCTIONAL_TEST_REPEAT": str(args.functional_test_repeat), + "FUNCTIONAL_TEST_CASES": args.functional_test_cases, + } + + trigger_pipeline( + gitlab_hostname, args.trigger_token, target_branch, pipeline_vars, dry_run=args.dry_run + ) + + +if __name__ == "__main__": + main() \ No newline at end of file