From f2b2988177695f702ebb6478fadeb157ad825a88 Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 1 May 2026 16:11:22 -0400 Subject: [PATCH 1/5] docs: add harness permissions to IAM policy and permissions guide Harnesses use imperative deployment (direct API calls, not CDK/CloudFormation), so the developer IAM principal needs harness CRUD + invoke permissions. This was missing, causing E2E harness tests to fail with 403 on CreateHarness. Added bedrock-agentcore:CreateHarness, GetHarness, UpdateHarness, DeleteHarness, ListHarnesses, and InvokeHarness to iam-policy-user.json and PERMISSIONS.md. --- docs/PERMISSIONS.md | 17 +++++++++++++++++ docs/policies/iam-policy-user.json | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md index 53ed2958d..650c2c158 100644 --- a/docs/PERMISSIONS.md +++ b/docs/PERMISSIONS.md @@ -39,6 +39,7 @@ Attach this to every IAM user or role that will run AgentCore CLI commands. The - `sts:GetCallerIdentity`, `cloudformation:DescribeStacks`, `tag:GetResources` for basic operations - `bedrock-agentcore:Invoke*`, `bedrock-agentcore:Get*`, `bedrock-agentcore:List*` for invoking agents and checking status +- Harness CRUD and invoke actions for `deploy`, `invoke`, and `status` when the project uses harnesses - Credential provider and token vault actions for `deploy` when the project uses identity features - CloudWatch Logs, X-Ray, and Application Signals actions for `logs`, `traces`, and observability setup - Bedrock actions for agent import and AI-assisted code generation (optional, see @@ -164,6 +165,7 @@ safely removed: | If your team does not use... | Remove from user policy | Remove from CFN execution policy | | ------------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Harnesses | `HarnessManagement` | _(no change)_ | | Container builds (CodeZip only) | _(no change)_ | `EcrContainerBuilds`, `CodeBuildContainerBuilds` | | MCP Lambda compute | _(no change)_ | `LambdaMcpAndCustomResources` (keep if using container builds, which need Lambda for custom resources) | | Agent import from Bedrock | `BedrockAgentImport` | _(no change)_ | @@ -335,6 +337,20 @@ Required for all deployment operations (`deploy`, `status`, `diff`). | `bedrock-agentcore:Evaluate` | `run evals` | Run on-demand evaluation against agent traces | | `bedrock-agentcore:UpdateOnlineEvaluationConfig` | `pause online-eval`, `resume online-eval` | Pause or resume online evaluation | +### Harness management + +Harnesses are deployed imperatively (direct API calls, not through CloudFormation), so harness CRUD permissions must be +on the developer's IAM principal, not just the CFN execution role. + +| Action | CLI Commands | Purpose | +| --------------------------------- | ---------------------------- | --------------------------------------------- | +| `bedrock-agentcore:CreateHarness` | `deploy` | Create a new harness | +| `bedrock-agentcore:GetHarness` | `deploy`, `status`, `invoke` | Get harness details and deployment state | +| `bedrock-agentcore:UpdateHarness` | `deploy` | Update an existing harness configuration | +| `bedrock-agentcore:DeleteHarness` | `deploy` | Delete a harness (during removal or teardown) | +| `bedrock-agentcore:ListHarnesses` | `status` | List harnesses in the account | +| `bedrock-agentcore:InvokeHarness` | `invoke` | Invoke a deployed harness (streaming) | + ### Identity and credential management | Action | CLI Commands | Purpose | @@ -428,6 +444,7 @@ actions used today. | `bedrock-agentcore:CreateOnlineEvaluationConfig`, `UpdateOnlineEvaluationConfig`, `DeleteOnlineEvaluationConfig`, `GetOnlineEvaluationConfig` | `AWS::BedrockAgentCore::OnlineEvaluationConfig` | | `bedrock-agentcore:CreatePolicyEngine`, `UpdatePolicyEngine`, `DeletePolicyEngine`, `GetPolicyEngine` | `AWS::BedrockAgentCore::PolicyEngine` | | `bedrock-agentcore:CreatePolicy`, `UpdatePolicy`, `DeletePolicy`, `GetPolicy` | `AWS::BedrockAgentCore::Policy` | +| `bedrock-agentcore:CreateHarness`, `UpdateHarness`, `DeleteHarness`, `GetHarness`, `ListHarnesses` | Imperative harness deployment (direct API) | ### IAM diff --git a/docs/policies/iam-policy-user.json b/docs/policies/iam-policy-user.json index d2467a134..3e3d428b3 100644 --- a/docs/policies/iam-policy-user.json +++ b/docs/policies/iam-policy-user.json @@ -55,6 +55,19 @@ ], "Resource": "*" }, + { + "Sid": "HarnessManagement", + "Effect": "Allow", + "Action": [ + "bedrock-agentcore:CreateHarness", + "bedrock-agentcore:GetHarness", + "bedrock-agentcore:UpdateHarness", + "bedrock-agentcore:DeleteHarness", + "bedrock-agentcore:ListHarnesses", + "bedrock-agentcore:InvokeHarness" + ], + "Resource": "*" + }, { "Sid": "IdentityCredentialManagement", "Effect": "Allow", From 649bbfcb23082da1f202d34e104d8642080b9d78 Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 1 May 2026 16:15:01 -0400 Subject: [PATCH 2/5] docs: remove misplaced harness row from CFN execution role table Harness permissions belong on the developer principal, not the CFN execution role. The user-policy "Harness management" section already documents these actions correctly. --- docs/PERMISSIONS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md index 650c2c158..e7e52eb13 100644 --- a/docs/PERMISSIONS.md +++ b/docs/PERMISSIONS.md @@ -444,7 +444,6 @@ actions used today. | `bedrock-agentcore:CreateOnlineEvaluationConfig`, `UpdateOnlineEvaluationConfig`, `DeleteOnlineEvaluationConfig`, `GetOnlineEvaluationConfig` | `AWS::BedrockAgentCore::OnlineEvaluationConfig` | | `bedrock-agentcore:CreatePolicyEngine`, `UpdatePolicyEngine`, `DeletePolicyEngine`, `GetPolicyEngine` | `AWS::BedrockAgentCore::PolicyEngine` | | `bedrock-agentcore:CreatePolicy`, `UpdatePolicy`, `DeletePolicy`, `GetPolicy` | `AWS::BedrockAgentCore::Policy` | -| `bedrock-agentcore:CreateHarness`, `UpdateHarness`, `DeleteHarness`, `GetHarness`, `ListHarnesses` | Imperative harness deployment (direct API) | ### IAM From b050dc15a2cd854f7a1f2e25672ab7cc4ca4b4d7 Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 1 May 2026 16:31:53 -0400 Subject: [PATCH 3/5] docs: add iam:PassRole for harness deployment The harness deployer passes a CDK-created execution role to the CreateHarness/UpdateHarness API, which requires iam:PassRole scoped with iam:PassedToService condition to bedrock-agentcore.amazonaws.com. --- docs/PERMISSIONS.md | 17 +++++++++-------- docs/policies/iam-policy-user.json | 11 +++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md index e7e52eb13..a46ecb5f2 100644 --- a/docs/PERMISSIONS.md +++ b/docs/PERMISSIONS.md @@ -342,14 +342,15 @@ Required for all deployment operations (`deploy`, `status`, `diff`). Harnesses are deployed imperatively (direct API calls, not through CloudFormation), so harness CRUD permissions must be on the developer's IAM principal, not just the CFN execution role. -| Action | CLI Commands | Purpose | -| --------------------------------- | ---------------------------- | --------------------------------------------- | -| `bedrock-agentcore:CreateHarness` | `deploy` | Create a new harness | -| `bedrock-agentcore:GetHarness` | `deploy`, `status`, `invoke` | Get harness details and deployment state | -| `bedrock-agentcore:UpdateHarness` | `deploy` | Update an existing harness configuration | -| `bedrock-agentcore:DeleteHarness` | `deploy` | Delete a harness (during removal or teardown) | -| `bedrock-agentcore:ListHarnesses` | `status` | List harnesses in the account | -| `bedrock-agentcore:InvokeHarness` | `invoke` | Invoke a deployed harness (streaming) | +| Action | CLI Commands | Purpose | +| --------------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `bedrock-agentcore:CreateHarness` | `deploy` | Create a new harness | +| `bedrock-agentcore:GetHarness` | `deploy`, `status`, `invoke` | Get harness details and deployment state | +| `bedrock-agentcore:UpdateHarness` | `deploy` | Update an existing harness configuration | +| `bedrock-agentcore:DeleteHarness` | `deploy` | Delete a harness (during removal or teardown) | +| `bedrock-agentcore:ListHarnesses` | `status` | List harnesses in the account | +| `bedrock-agentcore:InvokeHarness` | `invoke` | Invoke a deployed harness (streaming) | +| `iam:PassRole` | `deploy` | Pass the CDK-created execution role to the CreateHarness/UpdateHarness API. Scope with `iam:PassedToService: bedrock-agentcore.amazonaws.com` | ### Identity and credential management diff --git a/docs/policies/iam-policy-user.json b/docs/policies/iam-policy-user.json index 3e3d428b3..d5635a576 100644 --- a/docs/policies/iam-policy-user.json +++ b/docs/policies/iam-policy-user.json @@ -68,6 +68,17 @@ ], "Resource": "*" }, + { + "Sid": "HarnessPassRole", + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam::ACCOUNT_ID:role/*", + "Condition": { + "StringEquals": { + "iam:PassedToService": "bedrock-agentcore.amazonaws.com" + } + } + }, { "Sid": "IdentityCredentialManagement", "Effect": "Allow", From a86ba70cff112578df7e4d3641c671134a9c5def Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 1 May 2026 18:24:55 -0400 Subject: [PATCH 4/5] fix: update E2E test to match batch evaluation API migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - batchEvaluateId → batchEvaluationId (field renamed in API migration) - --lookback → --days for run eval (correct CLI flag) - Tool description recommendation test now expects ValidationException since the test agent has no tool traces (never calls search/calculator) --- e2e-tests/config-bundle-eval-rec.test.ts | 54 ++++++++++++------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/e2e-tests/config-bundle-eval-rec.test.ts b/e2e-tests/config-bundle-eval-rec.test.ts index 8151ac586..11ce1173c 100644 --- a/e2e-tests/config-bundle-eval-rec.test.ts +++ b/e2e-tests/config-bundle-eval-rec.test.ts @@ -368,7 +368,7 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' ); const json = parseJsonOutput(result.stdout) as Record; expect(json).toHaveProperty('success', true); - expect(json).toHaveProperty('batchEvaluateId'); + expect(json).toHaveProperty('batchEvaluationId'); expect(json.status).toBeDefined(); expect(json.status).not.toBe('FAILED'); }, @@ -446,7 +446,7 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' agentName, '--evaluator', 'Builtin.Faithfulness', - '--lookback', + '--days', '1', '--json', ]); @@ -537,34 +537,32 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' ); it.skipIf(!canRun)( - 'runs tool description recommendation via CLI', + 'runs tool description recommendation via CLI (expect validation error — agent has no tool traces)', async () => { - await retry( - async () => { - const result = await run([ - 'run', - 'recommendation', - '--type', - 'tool-description', - '--runtime', - agentName, - '--tools', - 'search:Searches the web for information', - '--tools', - 'calculator:Performs mathematical calculations', - '--lookback', - '1', - '--json', - ]); + // The test agent is a simple Strands agent invoked with "Say hello", which + // produces no tool calls. The API validates that requested tools appear in + // agent traces, so this correctly returns a ValidationException. + const result = await run([ + 'run', + 'recommendation', + '--type', + 'tool-description', + '--runtime', + agentName, + '--tools', + 'search:Searches the web for information', + '--tools', + 'calculator:Performs mathematical calculations', + '--lookback', + '1', + '--json', + ]); - expect(result.exitCode, `tool-desc recommendation failed: ${result.stdout}`).toBe(0); - const json = parseJsonOutput(result.stdout) as Record; - expect(json).toHaveProperty('success', true); - expect(json).toHaveProperty('recommendationId'); - }, - 6, - 30000 - ); + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', false); + expect(json.error).toBeDefined(); + expect(String(json.error)).toMatch(/not found in.*traces/i); }, 600000 ); From 8e97d24e15c1d5cc8d89a6895a8d895f1203951c Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 1 May 2026 18:25:58 -0400 Subject: [PATCH 5/5] Revert "fix: update E2E test to match batch evaluation API migration" This reverts commit a86ba70cff112578df7e4d3641c671134a9c5def. --- e2e-tests/config-bundle-eval-rec.test.ts | 54 ++++++++++++------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/e2e-tests/config-bundle-eval-rec.test.ts b/e2e-tests/config-bundle-eval-rec.test.ts index 11ce1173c..8151ac586 100644 --- a/e2e-tests/config-bundle-eval-rec.test.ts +++ b/e2e-tests/config-bundle-eval-rec.test.ts @@ -368,7 +368,7 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' ); const json = parseJsonOutput(result.stdout) as Record; expect(json).toHaveProperty('success', true); - expect(json).toHaveProperty('batchEvaluationId'); + expect(json).toHaveProperty('batchEvaluateId'); expect(json.status).toBeDefined(); expect(json.status).not.toBe('FAILED'); }, @@ -446,7 +446,7 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' agentName, '--evaluator', 'Builtin.Faithfulness', - '--days', + '--lookback', '1', '--json', ]); @@ -537,32 +537,34 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations' ); it.skipIf(!canRun)( - 'runs tool description recommendation via CLI (expect validation error — agent has no tool traces)', + 'runs tool description recommendation via CLI', async () => { - // The test agent is a simple Strands agent invoked with "Say hello", which - // produces no tool calls. The API validates that requested tools appear in - // agent traces, so this correctly returns a ValidationException. - const result = await run([ - 'run', - 'recommendation', - '--type', - 'tool-description', - '--runtime', - agentName, - '--tools', - 'search:Searches the web for information', - '--tools', - 'calculator:Performs mathematical calculations', - '--lookback', - '1', - '--json', - ]); + await retry( + async () => { + const result = await run([ + 'run', + 'recommendation', + '--type', + 'tool-description', + '--runtime', + agentName, + '--tools', + 'search:Searches the web for information', + '--tools', + 'calculator:Performs mathematical calculations', + '--lookback', + '1', + '--json', + ]); - expect(result.exitCode).toBe(1); - const json = parseJsonOutput(result.stdout) as Record; - expect(json).toHaveProperty('success', false); - expect(json.error).toBeDefined(); - expect(String(json.error)).toMatch(/not found in.*traces/i); + expect(result.exitCode, `tool-desc recommendation failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('recommendationId'); + }, + 6, + 30000 + ); }, 600000 );