Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/scripts/run-agentic-eval.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Stub runner: try known paths for an agentic-eval runner; fall back to skill validate
$repoRoot = (Resolve-Path "..\..").Path
Write-Output "Repo root: $repoRoot"

$pyRunner = Join-Path $repoRoot "skills\agentic-eval\scripts\run.py"
$nodeRunner = Join-Path $repoRoot "eng\run-agentic-eval.mjs"
Comment on lines +2 to +6

if (Test-Path $pyRunner) {
Write-Output "Found Python runner: $pyRunner. Executing..."
python $pyRunner
exit $LASTEXITCODE
} elseif (Test-Path $nodeRunner) {
Write-Output "Found Node runner: $nodeRunner. Executing..."
node $nodeRunner
exit $LASTEXITCODE
} else {
Write-Output "No agentic-eval runner found. Falling back to skill:validate (already run in workflow)."
Write-Output "This workflow is a draft/stub. Implement agentic-eval runner under skills/agentic-eval/scripts or eng/ to enable full evaluation."
exit 0
Comment on lines +16 to +19
}
Comment on lines +16 to +20
28 changes: 28 additions & 0 deletions .github/workflows/agentic-eval.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: Agentic Evaluation (PR)
on:
pull_request:
types: [opened, synchronize, reopened]

permissions:
contents: read

jobs:
agentic-eval:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '18'

Comment on lines +16 to +20
- name: Install dependencies
run: npm ci

- name: Validate skills
run: npm run skill:validate

- name: Run agentic-eval (stub runner)
run: pwsh -File .github/scripts/run-agentic-eval.ps1
Comment on lines +27 to +28
60 changes: 60 additions & 0 deletions governance/agent-policy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Agent policy for awesome-copilot
version: 1
name: default-agent-policy

# Allowed skills (explicit allow-list)
allowed_skills:
- acquire-codebase-knowledge
- ai-ready
- agent-governance
- audit-integrity
- agentic-eval
- agent-supply-chain
- arize-instrumentation

# Explicitly blocked or restricted skills
blocked_skills:
- auto-apply-code-changes-unreviewed

# Tool restrictions and capability tiers
tool_restrictions:
shell: 'allowed-with-review' # shell access requires review for write/change ops
network: 'restricted' # external network calls require allowlist
external_api_calls: 'whitelist-only'

# Secrets handling rules
secrets_handling:
forbid:
- writing_secrets_to_repo
- printing_secrets_in_logs
- committing_keys_or_tokens
require:
- use_vault_for_creds
- mask_secrets_in_outputs

# Non-negotiables enforced by CI / gates
non_negotiables:
- always_attach_evidence: true # include provenance for automated changes
- self_critique_required: true # agents must run self-eval before applying changes
- human_review_for_high_risk: true # high-risk changes must have human approval

# Evaluation gates and thresholds (1-10 scale)
evaluation_gates:
agentic_eval_threshold: 8 # must meet or exceed to auto-apply changes

# CI enforcement rules
ci_rules:
block_merge_on_policy_violation: true
fail_on_severity: 'high'

# Review and escalation process
review_process:
owner_roles:
- repo-admin
- security-owner
escalation: |
If an automated agent flags a high-severity policy violation, open an issue tagged 'agent-policy-violation' and notify owners. Do not merge until resolved.

# Notes
# - Keep this file under repository governance and sync with org-wide defaults.
# - Update allowed_skills as new vetted skills are approved.
12 changes: 12 additions & 0 deletions scripts/scan-repo.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
param(
[string]$RepoPath = '.'
)

Push-Location $RepoPath
if (!(Test-Path reports)) { New-Item -ItemType Directory -Path reports > $null }

# Run the built-in acquire-codebase-knowledge scan and save JSON output
python .\skills\acquire-codebase-knowledge\scripts\scan.py | Out-File -Encoding utf8 reports\acquire_scan.json
if ($?) { Write-Output "Scan complete: reports\acquire_scan.json" } else { Write-Error "Scan failed" }

Pop-Location
Comment on lines +5 to +12
Loading