From 5e6ba3ccbebf04e69d80da8688c816e598412901 Mon Sep 17 00:00:00 2001 From: toptiersalespro Date: Wed, 1 Jul 2026 18:35:59 -0700 Subject: [PATCH 1/2] Add agent governance policy, agentic-eval CI workflow, and scan artifacts (OODA plan) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/run-agentic-eval.ps1 | 20 ++++++++++ .github/workflows/agentic-eval.yml | 28 +++++++++++++ governance/agent-policy.yaml | 60 ++++++++++++++++++++++++++++ scripts/scan-repo.ps1 | 12 ++++++ 4 files changed, 120 insertions(+) create mode 100644 .github/scripts/run-agentic-eval.ps1 create mode 100644 .github/workflows/agentic-eval.yml create mode 100644 governance/agent-policy.yaml create mode 100644 scripts/scan-repo.ps1 diff --git a/.github/scripts/run-agentic-eval.ps1 b/.github/scripts/run-agentic-eval.ps1 new file mode 100644 index 000000000..031088726 --- /dev/null +++ b/.github/scripts/run-agentic-eval.ps1 @@ -0,0 +1,20 @@ +# Stub runner: try known paths for an agentic-eval runner; fall back to skill validate +$repoRoot = (Resolve-Path "..\..").Path +Write-Output "Repo root: $repoRoot" + +$pyRunner = Join-Path $repoRoot "skills\agentic-eval\scripts\run.py" +$nodeRunner = Join-Path $repoRoot "eng\run-agentic-eval.mjs" + +if (Test-Path $pyRunner) { + Write-Output "Found Python runner: $pyRunner. Executing..." + python $pyRunner + exit $LASTEXITCODE +} elseif (Test-Path $nodeRunner) { + Write-Output "Found Node runner: $nodeRunner. Executing..." + node $nodeRunner + exit $LASTEXITCODE +} else { + Write-Output "No agentic-eval runner found. Falling back to skill:validate (already run in workflow)." + Write-Output "This workflow is a draft/stub. Implement agentic-eval runner under skills/agentic-eval/scripts or eng/ to enable full evaluation." + exit 0 +} diff --git a/.github/workflows/agentic-eval.yml b/.github/workflows/agentic-eval.yml new file mode 100644 index 000000000..5c4e9340a --- /dev/null +++ b/.github/workflows/agentic-eval.yml @@ -0,0 +1,28 @@ +name: Agentic Evaluation (PR) +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +jobs: + agentic-eval: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install dependencies + run: npm ci + + - name: Validate skills + run: npm run skill:validate + + - name: Run agentic-eval (stub runner) + run: pwsh -File .github/scripts/run-agentic-eval.ps1 diff --git a/governance/agent-policy.yaml b/governance/agent-policy.yaml new file mode 100644 index 000000000..6d8cc74ae --- /dev/null +++ b/governance/agent-policy.yaml @@ -0,0 +1,60 @@ +# Agent policy for awesome-copilot +version: 1 +name: default-agent-policy + +# Allowed skills (explicit allow-list) +allowed_skills: + - acquire-codebase-knowledge + - ai-ready + - agent-governance + - audit-integrity + - agentic-eval + - agent-supply-chain + - arize-instrumentation + +# Explicitly blocked or restricted skills +blocked_skills: + - auto-apply-code-changes-unreviewed + +# Tool restrictions and capability tiers +tool_restrictions: + shell: 'allowed-with-review' # shell access requires review for write/change ops + network: 'restricted' # external network calls require allowlist + external_api_calls: 'whitelist-only' + +# Secrets handling rules +secrets_handling: + forbid: + - writing_secrets_to_repo + - printing_secrets_in_logs + - committing_keys_or_tokens + require: + - use_vault_for_creds + - mask_secrets_in_outputs + +# Non-negotiables enforced by CI / gates +non_negotiables: + - always_attach_evidence: true # include provenance for automated changes + - self_critique_required: true # agents must run self-eval before applying changes + - human_review_for_high_risk: true # high-risk changes must have human approval + +# Evaluation gates and thresholds (1-10 scale) +evaluation_gates: + agentic_eval_threshold: 8 # must meet or exceed to auto-apply changes + +# CI enforcement rules +ci_rules: + block_merge_on_policy_violation: true + fail_on_severity: 'high' + +# Review and escalation process +review_process: + owner_roles: + - repo-admin + - security-owner + escalation: | + If an automated agent flags a high-severity policy violation, open an issue tagged 'agent-policy-violation' and notify owners. Do not merge until resolved. + +# Notes +# - Keep this file under repository governance and sync with org-wide defaults. +# - Update allowed_skills as new vetted skills are approved. diff --git a/scripts/scan-repo.ps1 b/scripts/scan-repo.ps1 new file mode 100644 index 000000000..b2529c726 --- /dev/null +++ b/scripts/scan-repo.ps1 @@ -0,0 +1,12 @@ +param( + [string]$RepoPath = '.' +) + +Push-Location $RepoPath +if (!(Test-Path reports)) { New-Item -ItemType Directory -Path reports > $null } + +# Run the built-in acquire-codebase-knowledge scan and save JSON output +python .\skills\acquire-codebase-knowledge\scripts\scan.py | Out-File -Encoding utf8 reports\acquire_scan.json +if ($?) { Write-Output "Scan complete: reports\acquire_scan.json" } else { Write-Error "Scan failed" } + +Pop-Location From c0c41dec608a382b8be1f35050d3b4c11b200a91 Mon Sep 17 00:00:00 2001 From: toptiersalespro <213826078+toptiersalespro@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:50:56 -0700 Subject: [PATCH 2/2] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/agentic-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/agentic-eval.yml b/.github/workflows/agentic-eval.yml index 5c4e9340a..161bd6ee4 100644 --- a/.github/workflows/agentic-eval.yml +++ b/.github/workflows/agentic-eval.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Setup Node.js uses: actions/setup-node@v4