diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 14809a587..e1ff4acab 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -27,6 +27,7 @@ jobs: fail-fast: false matrix: cdk-source: [npm, main] + shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] steps: - uses: actions/checkout@v6 with: @@ -70,7 +71,7 @@ jobs: CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}) + - name: Run E2E tests (${{ matrix.cdk-source }}, shard ${{ matrix.shard }}) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -78,7 +79,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - run: npm run test:e2e + run: npx vitest run --project e2e --shard=${{ matrix.shard }} browser-tests: runs-on: ubuntu-latest environment: e2e-testing diff --git a/browser-tests/tests/traces.test.ts b/browser-tests/tests/traces.test.ts index 77c1b5ad4..0ac481f04 100644 --- a/browser-tests/tests/traces.test.ts +++ b/browser-tests/tests/traces.test.ts @@ -1,22 +1,38 @@ import { expect, sendMessage, test } from '../fixtures'; test.describe('Traces', () => { - test('traces panel shows trace after invocation', async ({ page }) => { + test('traces panel shows span tree after invocation', async ({ page }) => { await page.goto('/'); await sendMessage(page, 'Say hello'); - await page.getByRole('tab', { name: 'Traces' }).click(); + const resourcePanel = page.getByTestId('resource-panel'); + await expect(resourcePanel).toBeVisible({ timeout: 10_000 }); - const traceList = page.getByTestId('trace-list'); + const tracesTab = resourcePanel.getByRole('tab', { name: 'Traces' }); + await tracesTab.click(); + + // Wait for trace list to populate + const traceList = resourcePanel.getByTestId('traces-trace-list'); await expect(traceList).toBeVisible({ timeout: 30_000 }); + // Click the first trace const traceButton = traceList.getByRole('button').first(); - await expect(traceButton).toBeVisible({ timeout: 30_000 }); - + await expect(traceButton).toBeVisible({ timeout: 10_000 }); await traceButton.click(); - const spanRow = page.locator('[role="button"]').filter({ hasText: /.+/ }); - await expect(spanRow.first()).toBeVisible({ timeout: 10_000 }); + // Verify span tree renders + const spanTree = resourcePanel.getByTestId('traces-span-tree'); + await expect(spanTree).toBeVisible({ timeout: 10_000 }); + + // Verify tree has at least one span row with a name + const spanRows = spanTree.getByRole('button'); + await expect(spanRows.first()).toBeVisible(); + + // Click a span to open log panel + await spanRows.first().click(); + + const logPanel = resourcePanel.getByTestId('traces-log-panel'); + await expect(logPanel).toBeVisible({ timeout: 5_000 }); }); }); diff --git a/docs/TESTING.md b/docs/TESTING.md index 700ab3aae..9c70af6b3 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -415,6 +415,24 @@ Test configuration is in `vitest.config.ts` using Vitest projects: - Test timeout: 120 seconds - Hook timeout: 120 seconds +## Troubleshooting + +### `Cannot find module '@playwright/test'` + +Playwright is not installed. Run: + +```bash +npm install +``` + +### `browserType.launch: Executable doesn't exist` (Playwright browsers) + +Playwright browsers need to be downloaded after install. Run: + +```bash +npx playwright install chromium +``` + ## Integration Tests Integration tests require: diff --git a/e2e-tests/fixtures/import/cleanup_resources.py b/e2e-tests/fixtures/import/cleanup_resources.py index 0728b711e..429a36a1f 100644 --- a/e2e-tests/fixtures/import/cleanup_resources.py +++ b/e2e-tests/fixtures/import/cleanup_resources.py @@ -12,7 +12,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from common import REGION, RESOURCES_FILE, get_control_client, get_account_id +from common import REGION, RESOURCE_SUFFIX, RESOURCES_FILE, get_control_client, get_account_id import boto3 @@ -22,8 +22,12 @@ def cleanup_s3_code_objects(): account_id = get_account_id() bucket_name = f"bugbash-agentcore-code-{account_id}-{REGION}" s3 = boto3.client("s3", region_name=REGION) + prefix = f"bugbash-{RESOURCE_SUFFIX}/" if RESOURCE_SUFFIX else "" try: - resp = s3.list_objects_v2(Bucket=bucket_name) + list_args = {"Bucket": bucket_name} + if prefix: + list_args["Prefix"] = prefix + resp = s3.list_objects_v2(**list_args) objects = resp.get("Contents", []) if not objects: return @@ -31,7 +35,7 @@ def cleanup_s3_code_objects(): Bucket=bucket_name, Delete={"Objects": [{"Key": o["Key"]} for o in objects]}, ) - print(f"Deleted {len(objects)} object(s) from s3://{bucket_name}") + print(f"Deleted {len(objects)} object(s) from s3://{bucket_name}/{prefix}") except Exception as e: print(f"Could not clean up S3 objects: {e}") diff --git a/e2e-tests/fixtures/import/common.py b/e2e-tests/fixtures/import/common.py index 369ec0bb0..c786c0c01 100644 --- a/e2e-tests/fixtures/import/common.py +++ b/e2e-tests/fixtures/import/common.py @@ -8,9 +8,11 @@ import boto3 REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" +RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "") SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) APP_DIR = os.path.join(SCRIPT_DIR, "app") -RESOURCES_FILE = os.path.join(SCRIPT_DIR, "bugbash-resources.json") +_resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json" +RESOURCES_FILE = os.path.join(SCRIPT_DIR, _resources_name) INLINE_POLICY_NAME = "bugbash-agentcore-permissions" @@ -35,6 +37,8 @@ def upload_code(prefix="bugbash"): """Zip APP_DIR and upload to S3. Returns (bucket, s3_key).""" bucket_name = get_code_bucket() s3 = boto3.client("s3", region_name=REGION) + if RESOURCE_SUFFIX: + prefix = f"{prefix}-{RESOURCE_SUFFIX}" # Create zip of app directory with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: diff --git a/e2e-tests/import-gateway.test.ts b/e2e-tests/import-gateway.test.ts index 2aea04f02..fd80ae967 100644 --- a/e2e-tests/import-gateway.test.ts +++ b/e2e-tests/import-gateway.test.ts @@ -43,6 +43,7 @@ describe.sequential('e2e: import gateway', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'setup_gateway.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -50,7 +51,7 @@ describe.sequential('e2e: import gateway', () => { ); } - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; gatewayArn = resources.gateway!.arn; @@ -80,6 +81,7 @@ describe.sequential('e2e: import gateway', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ diff --git a/e2e-tests/import-resources.test.ts b/e2e-tests/import-resources.test.ts index d51cbffac..72d9c253a 100644 --- a/e2e-tests/import-resources.test.ts +++ b/e2e-tests/import-resources.test.ts @@ -54,6 +54,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', script], fixtureDir, { AWS_REGION: region, DEFAULT_EVALUATOR_MODEL, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -63,7 +64,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { } // 2. Read resource ARNs from bugbash-resources.json - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; runtimeArn = resources['runtime-basic']!.arn; memoryArn = resources['memory-full']!.arn; @@ -102,6 +103,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ diff --git a/package-lock.json b/package-lock.json index 45ad45ed3..a861d966d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "@aws-sdk/client-sts": "^3.893.0", "@aws-sdk/client-xray": "^3.1003.0", "@aws-sdk/credential-providers": "^3.893.0", - "@aws/agent-inspector": "0.2.1", + "@aws/agent-inspector": "0.3.0", "@commander-js/extra-typings": "^14.0.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-metrics-otlp-http": "^0.214.0", @@ -2903,9 +2903,9 @@ } }, "node_modules/@aws/agent-inspector": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@aws/agent-inspector/-/agent-inspector-0.2.1.tgz", - "integrity": "sha512-kyL6RBcTj1hYIchtrHDlDyeqm2viVYMBxhZKVn8wJn058YhI52GIDuUFlKD1avd57X+LJKlHr5VcKvBZp7Sg6A==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/agent-inspector/-/agent-inspector-0.3.0.tgz", + "integrity": "sha512-xD7QPr1WWkT9QWRWo6e9kq8kYxJLQ8egGscgSZ6jCyW3wNV5fcQ6THcAR/71hxxMFF2aleNUc3D8MoqgiS4DVw==", "license": "Apache-2.0", "dependencies": { "@ag-ui/core": "^0.0.52", diff --git a/package.json b/package.json index dda0501e2..e88a3c25e 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "@aws-sdk/client-sts": "^3.893.0", "@aws-sdk/client-xray": "^3.1003.0", "@aws-sdk/credential-providers": "^3.893.0", - "@aws/agent-inspector": "0.2.1", + "@aws/agent-inspector": "0.3.0", "@commander-js/extra-typings": "^14.0.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-metrics-otlp-http": "^0.214.0",