[TRTLLMINF-218][infra] Gate multi-GPU CI stages behind 'ci: full pre-merge approved' label

.github/workflows/bot-command.yml

@@ -55,9 +55,9 @@ jobs:
             "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label. Note: Does **NOT** update GitHub check status.\n\n" +
             "`--gpu-type \"A30, H100_PCIe\"` *(OPTIONAL)* : Only run the test stages on the specified GPU types. Examples: \"A30, H100_PCIe\". Note: Does **NOT** update GitHub check status.\n\n" +
             "`--test-backend \"pytorch, cpp\"` *(OPTIONAL)* : Skip test stages which don't match the specified backends. Only support [pytorch, cpp, tensorrt, triton]. Examples: \"pytorch, cpp\" (does not run test stages with tensorrt or triton backend). Note: Does **NOT** update GitHub pipeline status.\n\n" +
-            "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" +
+            "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Requires the `ci: full pre-merge approved` label on the PR (ask a member of NVIDIA/trt-llm-ci-approvers). Note: Does **NOT** update GitHub check status.\n\n" +
             "`--disable-multi-gpu-test ` *(OPTIONAL)* : Disable the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" +
-            "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline.\n\n" +
+            "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline. Requires the `ci: full pre-merge approved` label on the PR (ask a member of NVIDIA/trt-llm-ci-approvers).\n\n" +
             "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline. Requires the `ci: post-merge approved` PR label applied by an active member of `NVIDIA/trt-llm-ci-approvers`. The approval label remains in place when new commits are pushed.\n\n" +
             "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label.\n\n" +
             "`--detailed-log ` *(OPTIONAL)* : Enable flushing out all logs to the Jenkins console. This will significantly increase the log volume and may slow down the job.\n\n" +

.github/workflows/full-premerge-approval.yml

@@ -0,0 +1,222 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Guard Full Pre-Merge Approval Label
+
+on:
+  # Intentional: this runs static default-branch API logic only. It never checks
+  # out or executes PR code; it uses GitHub APIs to validate membership and
+  # manage this PR's approval label/comment.
+  pull_request_target:
+    types: [labeled]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  guard-full-premerge-approval:
+    concurrency:
+      group: premerge-label-guard-${{ github.event.pull_request.number }}
+      cancel-in-progress: false
+    # Keep a started job eligible to reach fail-closed cleanup after cancellation.
+    if: >-
+      always() &&
+      github.repository == 'NVIDIA/TensorRT-LLM' &&
+      github.event.action == 'labeled' &&
+      github.event.label.name == 'ci: full pre-merge approved'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Validate full pre-merge approver
+        id: validate
+        if: github.event.action == 'labeled'
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }}
+          result-encoding: string
+          script: |
+            const approvalLabel = 'ci: full pre-merge approved';
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const issueNumber = context.payload.pull_request.number;
+            let actor = context.payload.sender?.login || context.actor;
+            let labelEventId = '';
+            let timelineVerified = false;
+
+            try {
+              const events = await github.paginate(
+                github.rest.issues.listEventsForTimeline,
+                { owner, repo, issue_number: issueNumber, per_page: 100 }
+              );
+              const latestApprovalEvent = events
+                .filter(
+                  (event) =>
+                    event.event === 'labeled' &&
+                    event.label?.name?.toLowerCase() === approvalLabel.toLowerCase()
+                )
+                .at(-1);
+              if (latestApprovalEvent) {
+                actor = latestApprovalEvent.actor?.login || actor;
+                labelEventId = String(latestApprovalEvent.id);
+                timelineVerified = true;
+              } else {
+                core.warning('Could not identify the latest full pre-merge approval event.');
+              }
+            } catch (error) {
+              core.warning(
+                'Could not read the latest full pre-merge approval event: ' + error.message
+              );
+            }
+
+            core.setOutput('validated_actor', actor);
+            core.setOutput('label_event_id', labelEventId);
+            if (!timelineVerified) {
+              return 'false';
+            }
+            try {
+              const response = await github.request(
+                'GET /orgs/{org}/teams/{team_slug}/memberships/{username}',
+                {
+                  org: 'NVIDIA',
+                  team_slug: 'trt-llm-ci-approvers',
+                  username: actor,
+                }
+              );
+              const authorized = response.data.state === 'active';
+              console.log(
+                actor + ' active membership in NVIDIA/trt-llm-ci-approvers: ' + authorized
+              );
+              return authorized ? 'true' : 'false';
+            } catch (error) {
+              if (error.status === 404) {
+                console.log(
+                  actor + ' is not an active member of NVIDIA/trt-llm-ci-approvers.'
+                );
+              } else {
+                core.warning(
+                  'Could not verify full pre-merge approver ' + actor + ': ' + error.message
+                );
+              }
+              return 'false';
+            }
+
+      - name: Clear unauthorized full pre-merge approval
+        # Let a started job attempt fail-closed cleanup after normal cancellation.
+        if: always() && steps.validate.outputs.result != 'true'
+        uses: actions/github-script@v8
+        env:
+          VALIDATED_ACTOR: ${{ steps.validate.outputs.validated_actor }}
+          VALIDATED_LABEL_EVENT_ID: ${{ steps.validate.outputs.label_event_id }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const approvalLabel = 'ci: full pre-merge approved';
+            const actor =
+              process.env.VALIDATED_ACTOR ||
+              context.payload.sender?.login ||
+              context.actor;
+            const validatedEventId = process.env.VALIDATED_LABEL_EVENT_ID || '';
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const issueNumber = context.payload.pull_request.number;
+
+            try {
+              const pullRequest = await github.rest.pulls.get({
+                owner,
+                repo,
+                pull_number: issueNumber,
+              });
+              const labelIsPresent = pullRequest.data.labels.some(
+                (label) =>
+                  label.name?.toLowerCase() === approvalLabel.toLowerCase()
+              );
+              if (!labelIsPresent) {
+                console.log('Full pre-merge approval label is already absent; no cleanup needed.');
+                return;
+              }
+            } catch (error) {
+              core.warning(
+                'Could not read the current full pre-merge approval label state; continuing validation: ' +
+                error.message
+              );
+            }
+
+            try {
+              const events = await github.paginate(
+                github.rest.issues.listEventsForTimeline,
+                { owner, repo, issue_number: issueNumber, per_page: 100 }
+              );
+              const latestApprovalEvent = events
+                .filter(
+                  (event) =>
+                    event.event === 'labeled' &&
+                    event.label?.name?.toLowerCase() === approvalLabel.toLowerCase()
+                )
+                .at(-1);
+              const latestEventId = latestApprovalEvent
+                ? String(latestApprovalEvent.id)
+                : '';
+              const latestActor = latestApprovalEvent?.actor?.login || '';
+              if (!latestApprovalEvent) {
+                core.warning(
+                  'Could not identify the latest full pre-merge approval event; removing the label to fail closed.'
+                );
+              } else if (!latestActor) {
+                core.warning(
+                  'Could not identify the latest full pre-merge approval actor; removing the label to fail closed.'
+                );
+              } else if (!validatedEventId) {
+                core.warning(
+                  'The validation run did not bind an approval event; removing the label to fail closed.'
+                );
+              } else if (
+                latestEventId !== validatedEventId ||
+                latestActor !== actor
+              ) {
+                console.log(
+                  'A newer full pre-merge approval event was found; leaving it for its own validation run.'
+                );
+                return;
+              }
+            } catch (error) {
+              core.warning(
+                'Could not re-check the latest full pre-merge approval event; removing the label to fail closed: ' +
+                error.message
+              );
+            }
+
+            try {
+              await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: issueNumber,
+                name: approvalLabel,
+              });
+            } catch (error) {
+              if (error.status !== 404) {
+                throw error;
+              }
+            }
+
+            await github.rest.issues.createComment({
+              owner,
+              repo,
+              issue_number: issueNumber,
+              body:
+                'Removed the "' + approvalLabel + '" label because @' + actor +
+                ' could not be verified as an active member of ' +
+                'NVIDIA/trt-llm-ci-approvers. Ask a member of that team to apply it.',
+            });

docs/source/developer-guide/ci-overview.md

@@ -1,3 +1,20 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
 # Continuous Integration Overview
 
 This page explains how TensorRT‑LLM's CI is organized and how individual tests map to Jenkins stages. Most stages execute integration tests defined in YAML files, while unit tests run as part of a merge‑request pipeline. The sections below describe how to locate a test and trigger the stage that runs it.
@@ -9,7 +26,8 @@ This page explains how TensorRT‑LLM's CI is organized and how individual tests
 4. [Jenkins stage names](#jenkins-stage-names)
 5. [Finding the stage for a test](#finding-the-stage-for-a-test)
 6. [Waiving tests](#waiving-tests)
-7. [Triggering CI Best Practices](#triggering-ci-best-practices)
+7. [Multi-GPU Tests](#multi-gpu-tests)
+8. [Triggering CI Best Practices](#triggering-ci-best-practices)
 
 ## CI pipelines
 
@@ -105,6 +123,30 @@ full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_a
 Changes to `waives.txt` should include a bug link or brief explanation so other
 developers understand why the test is disabled.
 
+## Multi-GPU Tests
+
+Running multi-GPU tests (`--add-multi-gpu-test`, `--only-multi-gpu-test`) requires the
+`ci: full pre-merge approved` label on the PR. Without this label the
+`[Test-x86_64-Multi-GPU] Remote Run` and `[Test-SBSA-Multi-GPU] Remote Run` stages will
+fail with an explanatory error.
+
+To obtain the label, ask a member of `NVIDIA/trt-llm-ci-approvers` to apply it. Only
+members of that team can add the label; unauthorized additions are automatically removed
+by the `Guard Full Pre-Merge Approval Label` GitHub Actions workflow.
+
+Once the label is present, re-trigger CI with the same command you used
+originally. For example:
+
+```bash
+# Run the normal pipeline plus multi-GPU stages
+/bot run --add-multi-gpu-test
+
+# Run only multi-GPU stages
+/bot run --only-multi-gpu-test
+```
+
+Post-merge pipelines and GitLab MR builds are exempt from this label check.
+
 ## Triggering CI Best Practices
 
 ### Triggering Post-merge tests

jenkins/L0_MergeRequest.groovy

@@ -645,6 +645,53 @@ def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") {
     return result
 }
 
+// Gate multi-GPU stages behind 'ci: full pre-merge approved' label.
+// Uses trtllm_utils.validatePRLabelApproval() from the shared lib to verify
+// both label existence and that the labeler is an active team member.
+// Exempt: PostMerge pipelines and GitLab MR builds (no GITHUB_PR_API_URL).
+def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) {
+    if (!globalVars[GITHUB_PR_API_URL]) {
+        echo "[requireMultiGpuApprovalLabel] Skipping label check: not a GitHub PR (no GITHUB_PR_API_URL)"
+        return false
+    }
+    if (env.JOB_NAME ==~ /.*PostMerge.*/) {
+        echo "[requireMultiGpuApprovalLabel] Skipping label check: PostMerge pipeline is exempt"
+        return false
+    }
+
+    def prMatch = (globalVars[GITHUB_PR_API_URL] =~ /\/pulls?\/(\d+)/)
+    if (!prMatch) {
+        echo "[requireMultiGpuApprovalLabel] Could not extract PR number from ${globalVars[GITHUB_PR_API_URL]}. Failing open."
+        return false
+    }
+    def prNumber = prMatch[0][1]
+
+    def result = trtllm_utils.validatePRLabelApproval(pipeline, prNumber, "ci: full pre-merge approved")
+    if (!result.checkCompleted) {
+        // API error — fail-open: do not block CI if the label check itself fails
+        echo "[requireMultiGpuApprovalLabel] Label validation incomplete (${result.error}). Failing open."
+        return false
+    }
+    if (result.labelExists && result.authorized) {
+        return false
+    }
+
+    // Label missing or unauthorized — write description marker for wrapper
+    // to surface in PR comment, and return the block reason string.
+    def existingDesc = currentBuild.description ?: ""
+    currentBuild.description = existingDesc + (existingDesc ? "<br/>" : "") +
+        "<span data-multi-gpu-label-required='true'>" +
+        "Multi-GPU tests require label 'ci: full pre-merge approved'" +
+        "</span>"
+    def reason = !result.labelExists
+        ? "label 'ci: full pre-merge approved' is not present on this PR"
+        : "label 'ci: full pre-merge approved' was applied by '${result.actor}' who is not an active member of NVIDIA/trt-llm-ci-approvers"
+    def blockMsg = "${arch} Multi-GPU tests blocked: ${reason}. " +
+         "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, then re-trigger CI."
+    echo "[requireMultiGpuApprovalLabel] ${blockMsg}"
+    return blockMsg
+}
+
 def getMergeRequestChangedFileList(pipeline, globalVars) {
     def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/)
     if (env.alternativeTRT || isOfficialPostMergeJob) {
@@ -1568,6 +1615,19 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                     }
                 }
 
+                // Label gate: check before entering the Remote Run stage so a
+                // missing/unauthorized label shows as "Blocked" (not a Remote Run
+                // failure) and does not trigger fail-fast.
+                def x86LabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "x86_64")
+                if (x86LabelBlock) {
+                    stage("[Test-x86_64-Multi-GPU] Blocked") {
+                        catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                            error x86LabelBlock
+                        }
+                    }
+                    return
+                }
+
                 testStageName = "[Test-x86_64-Multi-GPU] Remote Run"
                 stage(testStageName) {
                     if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) {
@@ -1684,6 +1744,16 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                     }
                 }
 
+                def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA")
+                if (sbsaLabelBlock) {
+                    stage("[Test-SBSA-Multi-GPU] Blocked") {
+                        catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                            error sbsaLabelBlock
+                        }
+                    }
+                    return
+                }
+
                 testStageName = "[Test-SBSA-Multi-GPU] Remote Run"
                 stage(testStageName) {
                     if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) {