diff --git a/.cnb.yml b/.cnb.yml index 237d05b..f2ab5a8 100644 --- a/.cnb.yml +++ b/.cnb.yml @@ -22,60 +22,4 @@ master: echo "✅ 镜像推送成功!" echo "镜像地址: ${CNB_DOCKER_REGISTRY}/${CNB_REPO_SLUG_LOWERCASE}:${CNB_COMMIT_SHORT}" echo "最新镜像: ${CNB_DOCKER_REGISTRY}/${CNB_REPO_SLUG_LOWERCASE}:latest" - echo "========================================" - - # ======================================== - # 流水线 2: 同步代码到 GitHub 仓库 - # ======================================== - - name: "同步代码到 GitHub" - # 从环境配置仓库导入 GITHUB_TOKEN - imports: https://cnb.cool/ImAcaiy/envs/-/blob/main/envs.yml - stages: - - name: "配置 Git" - script: | - set -e - echo "========================================" - echo "🔄 开始同步到 GitHub" - echo "========================================" - - # 配置 Git 用户信息 - git config --global user.name "CNB Bot" - git config --global user.email "imacaiy@outlook.com" - - # 添加 GitHub remote(URL 会在推送时更新) - git remote add github https://github.com/acai1998/Automation_Platform.git || git remote set-url github https://github.com/acai1998/Automation_Platform.git - - echo "✅ Git 配置完成" - - - name: "推送到 GitHub" - script: | - set -e - - # 检查 GITHUB_TOKEN 是否设置 - if [ -z "$GITHUB_TOKEN" ]; then - echo "❌ 错误: GITHUB_TOKEN 环境变量未设置" - echo "请检查以下配置:" - echo "1. envs.yml 中是否已定义 GITHUB_TOKEN" - echo "2. Token 是否有 repo 权限" - echo "3. envs.yml 路径是否正确:https://cnb.cool/ImAcaiy/envs/-/blob/main/envs.yml" - exit 1 - fi - - # 使用 token 推送到 GitHub - GITHUB_URL="https://x-access-token:$GITHUB_TOKEN@github.com/acai1998/Automation_Platform.git" - - echo "📤 推送代码到 GitHub..." - echo "提交: ${CNB_COMMIT_SHA}" - echo "分支: ${CNB_BRANCH}" - echo "Token 长度: ${#GITHUB_TOKEN}" - - # 更新 GitHub remote URL 以包含 token - git remote set-url github ${GITHUB_URL} - - # 推送到 GitHub(不使用 force,避免触发分支保护规则) - git push github ${CNB_BRANCH} - - echo "========================================" - echo "✅ 同步成功!" - echo "🔗 GitHub 仓库: https://github.com/acai1998/Automation_Platform" - echo "========================================" + echo "========================================" \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 64f1e49..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,44 +0,0 @@ -## 描述 - -请简要描述此 PR 的更改。 - -## 相关 Issue - -关闭 #(Issue 号) - -## 修改类型 - -请选择相关选项: - -- [ ] Bug 修复(修复现有功能的非中断更改) -- [ ] 新功能(添加新功能的非中断更改) -- [ ] 重大更改(可能导致现有功能意外中断的修复或功能) -- [ ] 文档更新 - -## 修改清单 - -- [ ] 我已阅读 [CONTRIBUTING.md](../CONTRIBUTING.md) -- [ ] 我的代码遵循项目的代码风格 -- [ ] 我已进行了自我审查 -- [ ] 我已对我的代码进行了注释,特别是在复杂区域 -- [ ] 我已进行了相应的文档更改 -- [ ] 我的更改没有生成新的警告 -- [ ] 我已添加测试来证明我的修复/功能有效 -- [ ] 新的和现有的单元测试都通过了我的更改 - -## 测试 - -请描述你测试了这些更改的方式: - -- [ ] 前端类型检查:`npx tsc --noEmit -p tsconfig.json` -- [ ] 后端类型检查:`npx tsc --noEmit -p tsconfig.server.json` -- [ ] 本地开发测试:`npm run start` -- [ ] 其他测试(请描述) - -## 截图(如果适用) - -请添加相关的截图来演示你的更改。 - -## 其他信息 - -添加任何其他应该注意的信息。 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fa8943..c639303 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,20 +34,3 @@ jobs: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} DB_NAME: ${{ secrets.DB_NAME }} run: python .github/scripts/sync_cases.py - - test-typescript: - runs-on: ubuntu-latest - if: github.event_name == 'push' - steps: - - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - run: npm ci - - - name: Run TypeScript tests - run: npm run test diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml new file mode 100644 index 0000000..42b7a3f --- /dev/null +++ b/.github/workflows/github-ci.yml @@ -0,0 +1,135 @@ +name: CI/CD Pipeline + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + name: Lint & Type Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: TypeScript check (frontend) + run: npx tsc --noEmit -p tsconfig.json + + - name: TypeScript check (backend) + run: npx tsc --noEmit -p tsconfig.server.json + + test: + name: Test (${{ matrix.platform }} / Node ${{ matrix.node }}) + needs: lint + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + node: [18, 20, 22] + + runs-on: ${{ matrix.platform }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js ${{ matrix.node }} + uses: actions/setup-node@v4 + with: + node-version: '${{ matrix.node }}' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npx vitest run + + build: + name: Build Verification + needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build frontend + run: npm run build + + - name: Build backend + run: npm run server:build + + - name: Verify build output + run: | + test -f dist/index.html || (echo "Frontend build missing" && exit 1) + test -f dist/server/server/index.js || (echo "Backend build missing" && exit 1) + echo "Build verification passed" + + coverage: + name: Test Coverage + needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests with coverage + run: npx vitest run --coverage + + - name: Extract coverage percentage + id: coverage + run: | + PCT=$(node -e " + const fs = require('fs'); + const p = require('path'); + const dir = 'coverage'; + const files = fs.readdirSync(dir).filter(f => f === 'coverage-summary.json'); + if (!files.length) { console.error('No coverage-summary.json found'); process.exit(1); } + const data = JSON.parse(fs.readFileSync(p.join(dir, files[0]), 'utf8')); + console.log(data.total.lines.pct); + ") + echo "percentage=$PCT" >> "$GITHUB_OUTPUT" + echo "Coverage: $PCT%" + + - name: Upload coverage to Gist + if: github.event_name != 'pull_request' + uses: schneegans/dynamic-badges-action@v1.7.0 + with: + auth: ${{ secrets.GIST_TOKEN }} + gistID: ${{ secrets.COVERAGE_GIST_ID }} + filename: coverage.json + label: coverage + message: ${{ steps.coverage.outputs.percentage }}% + valColorRange: ${{ steps.coverage.outputs.percentage }} + minColorRange: 60 + maxColorRange: 90 + isError: ${{ steps.coverage.outputs.percentage < 60 }} diff --git a/.github/workflows/k6-smoke.yml b/.github/workflows/k6-smoke.yml new file mode 100644 index 0000000..580331d --- /dev/null +++ b/.github/workflows/k6-smoke.yml @@ -0,0 +1,45 @@ +name: k6 API Performance Smoke + +on: + workflow_dispatch: + schedule: + # Daily at Singapore/China 09:00. GitHub cron uses UTC. + - cron: '0 1 * * *' + +permissions: + contents: read + +jobs: + k6-smoke: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install k6 + run: | + sudo gpg -k + curl -s https://dl.k6.io/key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/k6-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install -y k6 + + - name: Run k6 API smoke test + working-directory: perf + env: + BASE_URL: ${{ vars.PERF_BASE_URL || secrets.PERF_BASE_URL }} + API_TOKEN: ${{ secrets.PERF_API_TOKEN }} + SMOKE_EMAIL: ${{ secrets.PERF_SMOKE_EMAIL }} + SMOKE_PASSWORD: ${{ secrets.PERF_SMOKE_PASSWORD }} + run: | + k6 run \ + --summary-export k6-summary.json \ + smoke.js + + - name: Upload k6 summary + if: always() + uses: actions/upload-artifact@v4 + with: + name: k6-summary + path: perf/k6-summary.json diff --git a/.github/workflows/sync-to-cnb.yml b/.github/workflows/sync-to-cnb.yml index 068062f..ae9b3f3 100644 --- a/.github/workflows/sync-to-cnb.yml +++ b/.github/workflows/sync-to-cnb.yml @@ -83,4 +83,4 @@ jobs: run: | echo "Syncing to GitHub..." git push https://x-access-token:${GITHUB_TOKEN}@github.com/acai1998/Automation_Platform.git HEAD:master - echo "GitHub sync completed!" + echo "GitHub sync completed!" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5489eab..38682a9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ __pycache__/ .env.example .mrules .catpaw/ +node_modules # 规则忽略文件 CLAUDE.md @@ -27,7 +28,8 @@ Jenkinsfile.optimized .markdownlint.yaml .markdownlint.json .docs\报错日志.txt - +.superpowers/* +.worktrees/ # Distribution / packaging .Python diff --git a/README.md b/README.md index 88149d7..45f5d40 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ TailwindCSS PM2 Socket.IO + Test Coverage

一个现代化的全栈自动化测试管理平台,用于管理测试用例、调度 Jenkins 执行任务、监控执行结果。平台专注于测试管理和调度,实际测试执行由 Jenkins 等外部系统完成。 diff --git a/docs/superpowers/specs/2026-05-24-unit-test-coverage-design.md b/docs/superpowers/specs/2026-05-24-unit-test-coverage-design.md new file mode 100644 index 0000000..6614b24 --- /dev/null +++ b/docs/superpowers/specs/2026-05-24-unit-test-coverage-design.md @@ -0,0 +1,351 @@ +# Unit Test Coverage Expansion Design + +## Status: Approved + +## Date: 2026-05-24 + +## Background + +The project currently has ~24% test coverage across 29 test files (19 backend, 12 frontend). Critical gaps include backend repositories (0%), backend routes (9.1%), auth middleware (untested), and frontend core components (~10%). This design outlines a phased approach to systematically expand test coverage. + +## Current Coverage Summary + +| Category | Covered | Total | Coverage | +|----------|---------|-------|----------| +| Backend Services | 6 | 19 | 31.6% | +| Backend Routes | 1 | 11 | 9.1% | +| Backend Middleware | 1 | 5 | 20% | +| Backend Repositories | 0 | 14 | 0% | +| Backend Utils | 5 | 10 | 50% | +| Frontend Pages | 8 | 19 | 42% | +| Frontend Components | 2 | 20+ | ~10% | +| Frontend Hooks | 1 | 5 | 20% | +| Frontend Services | 1 | 2 | 50% | + +## Decision + +Expand test coverage in 4 phases, prioritizing security-critical and data-access layers first. + +## Test Conventions + +- Framework: Vitest + React Testing Library + jsdom +- Backend tests: `test_case/backend/{category}/{name}.test.ts` +- Frontend tests: `test_case/frontend/{category}/{name}.test.tsx` +- Use `vi.mock()` for dependencies, `vi.hoisted()` for mock instances +- Use `describe()` / `it()` blocks, descriptive test names +- Backend route tests: test pure logic functions (validation, normalization) extracted from handlers +- Frontend component tests: render with QueryClientProvider wrapper, mock hooks and UI libs + +## Phase 1: Backend Security Core + +### 1.1 Auth Middleware (`server/middleware/auth.ts`) + +File: `test_case/backend/middleware/auth.test.ts` + +Test cases: +- `authenticate`: missing Authorization header → 401 +- `authenticate`: malformed token (no "Bearer " prefix) → 401 +- `authenticate`: invalid/expired token → 401 +- `authenticate`: valid token → sets `req.user`, calls `next()` +- `optionalAuth`: no header → passes through without user +- `optionalAuth`: valid token → sets `req.user` +- `optionalAuth`: invalid token → passes through without user +- `requireRole('admin')`: no `req.user` → 401 +- `requireRole('admin')`: wrong role → 403 +- `requireRole('admin')`: correct role → calls `next()` +- `requireRole('admin', 'tester')`: tester role → calls `next()` +- `requireAdmin`: delegates to `requireRole('admin')` +- `requireTester`: delegates to `requireRole('admin', 'tester', 'developer')` + +Mocks: `authService.verifyToken`, `logger` + +### 1.2 Auth Routes (`server/routes/auth.ts`) + +File: `test_case/backend/routes/auth.route.test.ts` + +Test cases: +- POST `/register`: missing fields → 400 +- POST `/register`: invalid email format → 400 +- POST `/register`: password too short → 400 +- POST `/register`: username too short → 400 +- POST `/register`: success → 201 with user data +- POST `/login`: missing credentials → 400 +- POST `/login`: success → 200 with token +- POST `/logout`: no auth → 401 +- POST `/logout`: success → 200 +- GET `/me`: no auth → 401 +- GET `/me`: success → 200 with user data +- POST `/forgot-password`: missing email → 400 +- POST `/reset-password`: missing token/password → 400 +- POST `/refresh`: missing token → 400 + +Mocks: `authService`, `authenticate` middleware + +### 1.3 UserRepository (`server/repositories/UserRepository.ts`) + +File: `test_case/backend/repositories/UserRepository.test.ts` + +Test cases: +- `findByEmail`: returns user when found, null when not +- `findByUsername`: returns user when found, null when not +- `findById`: returns user when found, null when not +- `findByResetToken`: returns user with valid non-expired token, null when expired +- `findByRememberToken`: returns user with matching token +- `createUser`: creates with default status='active', loginAttempts=0 +- `updateLoginAttempts`: updates count correctly +- `lockUser` / `unlockUser`: updates lock fields +- `updateLastLogin`: updates timestamp +- `setResetToken`: stores token and expiry +- `resetPassword`: updates hash, clears token +- `setRememberToken`: stores/clears token + +Mocks: TypeORM `Repository` (findOne, update, create, save, createQueryBuilder) + +### 1.4 TestCaseRepository (`server/repositories/TestCaseRepository.ts`) + +File: `test_case/backend/repositories/TestCaseRepository.test.ts` + +Test cases: +- `findById`: returns case when found, null when not +- `findByName`: returns case when found +- `findByScriptPath`: returns case when found +- `findAll`: applies type/priority filters, pagination +- `createTestCase`: handles JSON config and tags parsing +- `updateTestCaseSafe`: validates JSON fields before update +- `deleteTestCase`: deletes by ID +- `count`: handles comma-separated multi-value filters +- `findAllWithUser`: joins user data +- `getDistinctOwners` / `getDistinctModules`: returns unique values +- `createTestCasesBatch`: batch insert in transaction +- `updateTestCasesBatch`: batch update in transaction +- `deleteTestCasesBatch`: validates existence before delete + +Mocks: TypeORM `Repository`, `DataSource`, `QueryRunner` + +## Phase 2: Backend Routes + +### 2.1 Cases Routes (`server/routes/cases.ts`) + +File: `test_case/backend/routes/cases.route.test.ts` + +Test cases: +- GET `/`: default pagination (limit=20, offset=0) +- GET `/`: custom pagination with limit cap at 500 +- GET `/`: projectId NaN → 400 +- GET `/`: filters (module, enabled, type, search, priority, owner) +- GET `/`: database error → 500 +- Response mapping: snake_case to camelCase conversion + +Mocks: `TestCaseRepository`, `AppDataSource`, `logger` + +### 2.2 Executions Routes (`server/routes/executions.ts`) + +File: `test_case/backend/routes/executions.route.test.ts` + +Test cases: +- POST `/callback`: missing executionId → 400 +- POST `/callback`: missing status → 400 +- POST `/callback`: missing results array → 400 +- POST `/callback`: success → 200 +- POST `/:id/start`: invalid ID → 400 +- POST `/:id/start`: success → 200 +- GET `/test-runs`: pagination and filter params + +Mocks: `executionService`, `query`, `logger` + +### 2.3 Dashboard Routes (`server/routes/dashboard.ts`) + +File: `test_case/backend/routes/dashboard.route.test.ts` + +Test cases: +- GET `/stats`: success → 200 with stats object +- GET `/stats`: service error → 500 +- GET `/today-execution`: success → 200 +- GET `/trend`: days parameter validation (1-365) +- GET `/trend`: invalid days → 400 +- Cache-Control headers set correctly + +Mocks: `dashboardService`, `dailySummaryScheduler`, `logger` + +### 2.4 Jenkins Routes (`server/routes/jenkins.ts`) + +File: `test_case/backend/routes/jenkins.route.test.ts` + +Test cases: +- POST `/trigger`: missing jobId → 400 +- POST `/trigger`: success → 200 +- GET `/status/:id`: invalid ID → 400 +- GET `/status/:id`: not found → 404 +- GET `/config`: returns configuration + +Mocks: `jenkinsService`, `authenticate`, `requireTester` + +## Phase 3: Frontend Hooks + Core Components + +### 3.1 useTasks Hook (`src/hooks/useTasks.ts`) + +File: `test_case/frontend/hooks/useTasks.test.ts` + +Test cases: +- `runWithConcurrencyLimit`: empty array → empty result +- `runWithConcurrencyLimit`: respects concurrency limit +- `runWithConcurrencyLimit`: handles errors in individual items +- `buildAuthHeaders`: includes token when present +- `buildAuthHeaders`: omits token when absent +- `useTasks`: constructs correct query URL with params +- `useRunTask`: sends POST request with correct body +- `useUpdateTaskStatus`: optimistic update and rollback on error +- `useDeleteTask`: optimistic removal and rollback on error +- `useBatchRunTask`: respects concurrency limit + +Mocks: `fetch`, `getToken`, `@tanstack/react-query` + +### 3.2 useExecutions Hook (`src/hooks/useExecutions.ts`) + +File: `test_case/frontend/hooks/useExecutions.test.ts` + +Test cases: +- `useTestRuns`: constructs query with filters +- `useTestRunDetail`: polls every 3s for running status +- `useTestRunDetail`: stops polling for terminal status +- `useTestRunResults`: includes auth token in fetch +- `useJenkinsHealthStatus`: returns connected:false on error +- `useStaleExecutionSummary`: correct query params + +Mocks: `request`, `fetch`, `@tanstack/react-query` + +### 3.3 ProtectedRoute Component (`src/components/ProtectedRoute.tsx`) + +File: `test_case/frontend/components/ProtectedRoute.test.tsx` + +Test cases: +- loading state → renders spinner +- unauthenticated → redirects to `/login` +- unauthenticated → encodes current path as returnUrl +- authenticated → renders children + +Mocks: `useAuth`, `useLocation`, `wouter/Redirect` + +### 3.4 ErrorBoundary Component (`src/components/ErrorBoundary.tsx`) + +File: `test_case/frontend/components/ErrorBoundary.test.tsx` + +Test cases: +- no error → renders children normally +- child throws → renders fallback UI +- child throws → logs error + +Mocks: console.error (suppress) + +## Phase 4: Frontend Components + Utilities + +### 4.1 Sidebar Component (`src/components/Sidebar.tsx`) + +File: `test_case/frontend/components/Sidebar.test.tsx` + +Test cases: +- renders all navigation items +- highlights active item based on current route +- expands/collapses child sections +- logout button calls logout and navigates + +Mocks: `useLocation`, `useAuth`, `useNavCollapse`, `useAiGeneration`, `createPortal` + +### 4.2 Layout Component (`src/components/Layout.tsx`) + +File: `test_case/frontend/components/Layout.test.tsx` + +Test cases: +- renders Sidebar and children +- applies correct layout structure + +Mocks: `Sidebar` + +### 4.3 useCases Hook (`src/hooks/useCases.ts`) + +File: `test_case/frontend/hooks/useCases.test.ts` + +Test cases: +- `useCases`: constructs query with filters +- `useCreateCase`: sends POST and invalidates cache +- `useUpdateCase`: sends PATCH and invalidates cache +- `useDeleteCase`: sends DELETE and invalidates cache + +Mocks: `request`, `@tanstack/react-query` + +### 4.4 authApi Service (`src/services/authApi.ts`) + +File: `test_case/frontend/services/authApi.test.ts` + +Test cases: +- `getToken`: returns token from localStorage +- `setToken`: stores token in localStorage +- `clearToken`: removes token from localStorage +- `login`: sends POST and stores token +- `logout`: sends POST and clears token + +Mocks: `localStorage`, `fetch` + +### 4.5 ServiceError Utility (`server/utils/ServiceError.ts`) + +File: `test_case/backend/utils/ServiceError.test.ts` + +Test cases: +- constructs with message and code +- preserves stack trace +- instanceof Error check + +### 4.6 Type Validation Utility (`server/utils/type-validation.ts`) + +File: `test_case/backend/utils/typeValidation.test.ts` + +Test cases: +- validates email format +- validates password strength +- validates required fields +- sanitizes string inputs + +## File Structure + +``` +test_case/ +├── backend/ +│ ├── middleware/ +│ │ ├── authRateLimiter.test.ts (existing) +│ │ └── auth.test.ts (Phase 1) +│ ├── repositories/ +│ │ ├── UserRepository.test.ts (Phase 1) +│ │ └── TestCaseRepository.test.ts (Phase 1) +│ ├── routes/ +│ │ ├── tasks.route.test.ts (existing) +│ │ ├── auth.route.test.ts (Phase 1) +│ │ ├── cases.route.test.ts (Phase 2) +│ │ ├── executions.route.test.ts (Phase 2) +│ │ ├── dashboard.route.test.ts (Phase 2) +│ │ └── jenkins.route.test.ts (Phase 2) +│ └── utils/ +│ ├── ServiceError.test.ts (Phase 4) +│ └── typeValidation.test.ts (Phase 4) +├── frontend/ +│ ├── components/ +│ │ ├── ThemeToggle.test.tsx (existing) +│ │ ├── AiCaseSidebar.test.tsx (existing) +│ │ ├── ProtectedRoute.test.tsx (Phase 3) +│ │ ├── ErrorBoundary.test.tsx (Phase 3) +│ │ ├── Sidebar.test.tsx (Phase 4) +│ │ └── Layout.test.tsx (Phase 4) +│ ├── hooks/ +│ │ ├── useExecuteCase.test.tsx (existing) +│ │ ├── useTasks.test.ts (Phase 3) +│ │ ├── useExecutions.test.ts (Phase 3) +│ │ └── useCases.test.ts (Phase 4) +│ └── services/ +│ └── authApi.test.ts (Phase 4) +``` + +## Expected Outcome + +- New test files: ~20 +- Target coverage after completion: ~50-60% +- Each phase is independently mergeable +- No changes to production code required diff --git a/perf/endpoints.json b/perf/endpoints.json new file mode 100644 index 0000000..49dbc63 --- /dev/null +++ b/perf/endpoints.json @@ -0,0 +1,39 @@ +[ + { + "name": "health", + "method": "GET", + "path": "/api/health", + "expectedStatus": 200 + }, + { + "name": "login", + "method": "POST", + "path": "/api/auth/login", + "body": { + "email": "${SMOKE_EMAIL}", + "password": "${SMOKE_PASSWORD}" + }, + "expectedStatus": 200, + "requiresCredentials": true, + "captureToken": true + }, + { + "name": "current_user", + "method": "GET", + "path": "/api/auth/me", + "expectedStatus": 200, + "requiresToken": true + }, + { + "name": "dashboard_stats", + "method": "GET", + "path": "/api/dashboard/stats", + "expectedStatus": 200 + }, + { + "name": "cases_list", + "method": "GET", + "path": "/api/cases?limit=10&offset=0", + "expectedStatus": 200 + } +] diff --git a/perf/smoke.js b/perf/smoke.js new file mode 100644 index 0000000..6c0c9f5 --- /dev/null +++ b/perf/smoke.js @@ -0,0 +1,112 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { SharedArray } from 'k6/data'; + +const BASE_URL = (__ENV.BASE_URL || '').replace(/\/$/, ''); +const API_TOKEN = __ENV.API_TOKEN || ''; +const SMOKE_EMAIL = __ENV.SMOKE_EMAIL || ''; +const SMOKE_PASSWORD = __ENV.SMOKE_PASSWORD || ''; +let authToken = API_TOKEN; + +if (!BASE_URL) { + throw new Error('BASE_URL is required'); +} + +const endpoints = new SharedArray('endpoints', function () { + const parsed = JSON.parse(open('./endpoints.json')); + + if (!Array.isArray(parsed)) { + throw new Error('endpoints.json must contain an array'); + } + + return parsed; +}); + +export const options = { + vus: 1, + iterations: 10, + thresholds: { + http_req_failed: ['rate<0.01'], + http_req_duration: ['p(95)<800'], + checks: ['rate>0.99'], + }, +}; + +function resolveBody(body) { + if (!body) { + return {}; + } + + const text = JSON.stringify(body) + .replaceAll('${SMOKE_EMAIL}', SMOKE_EMAIL) + .replaceAll('${SMOKE_PASSWORD}', SMOKE_PASSWORD); + + return JSON.parse(text); +} + +function shouldSkip(api) { + if (api.requiresToken && !authToken) { + return true; + } + + if (api.requiresCredentials && (!SMOKE_EMAIL || !SMOKE_PASSWORD)) { + return true; + } + + return false; +} + +function captureAuthToken(api, response) { + if (!api.captureToken || response.status !== api.expectedStatus) { + return; + } + + const body = response.json(); + if (body && typeof body.token === 'string') { + authToken = body.token; + } +} + +export default function () { + for (const api of endpoints) { + if (shouldSkip(api)) { + continue; + } + + const method = String(api.method || 'GET').toUpperCase(); + const url = `${BASE_URL}${api.path}`; + const headers = { + 'Content-Type': 'application/json', + }; + + if (authToken) { + headers.Authorization = `Bearer ${authToken}`; + } + + const params = { + headers, + tags: { api: api.name }, + }; + const body = JSON.stringify(resolveBody(api.body)); + let response; + + if (method === 'POST') { + response = http.post(url, body, params); + } else if (method === 'PUT') { + response = http.put(url, body, params); + } else if (method === 'DELETE') { + response = http.del(url, null, params); + } else { + response = http.get(url, params); + } + + check(response, { + [`${api.name} status is ${api.expectedStatus}`]: (r) => r.status === api.expectedStatus, + [`${api.name} response time < 800ms`]: (r) => r.timings.duration < 800, + }); + + captureAuthToken(api, response); + + sleep(1); + } +} diff --git a/server/repositories/DashboardRepository.ts b/server/repositories/DashboardRepository.ts index b7f3a6f..1c556d1 100644 --- a/server/repositories/DashboardRepository.ts +++ b/server/repositories/DashboardRepository.ts @@ -4,108 +4,40 @@ import { BaseRepository } from './BaseRepository'; import { ServiceError } from '../utils/ServiceError'; import logger from '../utils/logger'; import { LOG_CONTEXTS } from '../config/logging'; +import { + buildContinuousTrendData, + calculatePercentage, + calculateSuccessRate, + formatLocalDate, + generateDateRange, + hasTrendExecutionData, + logDashboard, + normalizeDailySummaryRows, + parseSafeFloat, + parseSafeInt, + parseStatsResult, +} from './DashboardRepositoryUtils'; +import type { + ActiveCasesStats, + DailySummaryData, + DashboardStats, + DateStats, + ExecutionStats, + RecentRun, + SummaryStats, + TodayExecution, + TrendDebugInfo, + TrendDebugSourceStats, +} from './DashboardRepositoryTypes'; +export type { + DailySummaryData, + DashboardStats, + RecentRun, + TodayExecution, + TrendDebugInfo, + TrendDebugSourceStats, +} from './DashboardRepositoryTypes'; -export interface DashboardStats { - totalCases: number; - todayRuns: number; - todaySuccessRate: number; - runningTasks: number; -} - -export interface TodayExecution { - total: number; - passed: number; - failed: number; - skipped: number; -} - -export interface DailySummaryData { - date: string; - totalExecutions: number; - passedCases: number; - failedCases: number; - skippedCases: number; - successRate: number; -} - -export interface RecentRun { - id: number; - suiteName?: string; - status: string; - duration: number; - startTime?: Date; - totalCases: number; - passedCases: number; - failedCases: number; - executedBy?: string; - executedById?: number; -} - -export interface TrendDebugSourceStats { - source: 'daily_summary' | 'test_run' | 'task_execution'; - rowCount: number; - daysWithData: number; - totalExecutions: number; - passedCases: number; - failedCases: number; - skippedCases: number; - latestDate: string | null; -} - -export interface TrendDebugInfo { - days: number; - dateRange: { - startDate: string; - endDate: string; - }; - sources: TrendDebugSourceStats[]; -} - -/** - * 执行统计查询结果接口 - */ -interface ExecutionStats { - total: string; - passed: string; - failed: string; - skipped: string; -} - -/** - * 汇总数据查询结果接口 - */ -interface SummaryStats { - totalExecutions: string; - totalCasesRun: string; - passedCases: string; - failedCases: string; - skippedCases: string; - avgDuration: string; -} - -/** - * 活跃用例数查询结果接口 - */ -interface ActiveCasesStats { - count: string; -} - -/** - * 日期统计查询结果接口 - */ -interface DateStats { - summaryDate: string; - totalExecutions: string; - totalCasesRun: string; - passedCases: string; - failedCases: string; - skippedCases: string; - avgDuration: string; -} - -/** - * 仪表盘数据 Repository - */ export class DashboardRepository extends BaseRepository { private testCaseRepository: Repository; private taskExecutionRepository: Repository; @@ -120,225 +52,6 @@ export class DashboardRepository extends BaseRepository { this.userRepository = dataSource.getRepository(User); } - /** - * 安全的整数解析方法 - * @param value 要解析的值 - * @param defaultValue 默认值 - * @returns 解析后的整数 - */ - private parseSafeInt(value: string | number | null | undefined, defaultValue: number = 0): number { - if (value === null || value === undefined) { - return defaultValue; - } - - const parsed = typeof value === 'string' ? parseInt(value, 10) : value; - return isNaN(parsed) ? defaultValue : parsed; - } - - /** - * 安全的浮点数解析方法 - * @param value 要解析的值 - * @param defaultValue 默认值 - * @returns 解析后的浮点数 - */ - private parseSafeFloat(value: string | number | null | undefined, defaultValue: number = 0): number { - if (value === null || value === undefined) { - return defaultValue; - } - - const parsed = typeof value === 'string' ? parseFloat(value) : value; - return isNaN(parsed) ? defaultValue : parsed; - } - - /** - * 安全的百分比计算 - * @param current 当前值 - * @param previous 之前值 - * @returns 计算后的百分比,如果无法计算返回null - */ - private calculatePercentage(current: number, previous: number): number | null { - if (previous <= 0) return null; - return Math.round(((current - previous) / previous) * 10000) / 100; - } - - /** - * 本地日期格式化(YYYY-MM-DD) - * 避免 toISOString() 的 UTC 转换导致日期偏移 - */ - private formatLocalDate(date: Date): string { - const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, '0'); - const day = String(date.getDate()).padStart(2, '0'); - return `${year}-${month}-${day}`; - } - - /** - * 日期范围生成器 - * 使用生成器减少内存占用,避免创建大数组 - * @param days 天数 - * @returns 日期生成器 - */ - private *generateDateRange(days: number): Generator { - const today = new Date(); - for (let i = 1; i <= days; i++) { - const date = new Date(today); - date.setDate(date.getDate() - i); - yield this.formatLocalDate(date); - } - } - - /** - * 安全的数据库查询执行方法 - * @param query 执行的查询函数 - * @param operation 操作描述 - * @param context 上下文信息 - * @returns 查询结果 - */ - private async executeQuery( - query: () => Promise, - operation: string, - context?: Record - ): Promise { - try { - return await query(); - } catch (error) { - logger.errorLog(error, `Failed to execute ${operation}`, { - operation, - context, - }); - - throw new ServiceError( - `Failed to execute ${operation}`, - error instanceof Error ? error : new Error(String(error)), - 500, - 'DATA_ACCESS_ERROR', - { operation, context } - ); - } - } - - /** - * 安全的统计计算方法 - * 统一处理统计查询结果的解析和验证 - * @param result 查询结果数组 - * @param defaultValue 默认值 - * @returns 解析后的统计数据 - */ - private parseStatsResult>( - result: T[], - defaultValue: T - ): T { - return result[0] || defaultValue; - } - - /** - * 统一规范趋势数据类型,避免数据库驱动返回 string 造成前端校验失败 - * 成功率统一通过 passed/total 重新计算,避免依赖可能滞后的 success_rate 字段 - */ - private normalizeDailySummaryRows( - rows: Array<{ - date: string; - totalExecutions: string | number | null; - passedCases: string | number | null; - failedCases: string | number | null; - skippedCases: string | number | null; - successRate: string | number | null; - }> - ): DailySummaryData[] { - return rows.map((row) => { - const passedCases = this.parseSafeInt(row.passedCases, 0); - const failedCases = this.parseSafeInt(row.failedCases, 0); - const skippedCases = this.parseSafeInt(row.skippedCases, 0); - const totalCases = passedCases + failedCases + skippedCases; - const successRate = totalCases > 0 - ? Math.round((passedCases / totalCases) * 10000) / 100 - : 0; - - return { - date: row.date, - totalExecutions: this.parseSafeInt(row.totalExecutions, 0), - passedCases, - failedCases, - skippedCases, - successRate, - }; - }); - } - - /** - * 补齐近 N 天趋势数据,确保每天都有一条记录 - */ - private buildContinuousTrendData(days: number, rows: DailySummaryData[]): DailySummaryData[] { - const rowMap = new Map(rows.map((row) => [row.date, row])); - const continuousData: DailySummaryData[] = []; - - for (const date of this.generateDateRange(days)) { - continuousData.push( - rowMap.get(date) ?? { - date, - totalExecutions: 0, - passedCases: 0, - failedCases: 0, - skippedCases: 0, - successRate: 0, - } - ); - } - - return continuousData.reverse(); - } - - /** - * 判断趋势数据是否包含有效运行记录 - */ - private hasTrendExecutionData(rows: DailySummaryData[]): boolean { - return rows.some((row) => - row.totalExecutions > 0 || - row.passedCases > 0 || - row.failedCases > 0 || - row.skippedCases > 0 - ); - } - - /** - * 计算成功率 - * @param passed 通过数量 - * @param total 总数量 - * @returns 成功率百分比 - */ - private calculateSuccessRate(passed: number, total: number): number { - if (total <= 0) return 0; - return Math.round((passed / total) * 10000) / 100; - } - - /** - * 统一的日志记录方法 - * @param level 日志级别 - * @param message 日志消息 - * @param data 日志数据 - * @param context 日志上下文 - */ - private logDashboard( - level: 'debug' | 'info' | 'warn' | 'error', - message: string, - data?: Record, - context: string = LOG_CONTEXTS.DASHBOARD, - error?: unknown - ) { - if (level === 'debug') { - logger.debug(message, data, context); - } else if (level === 'info') { - logger.info(message, data, context); - } else if (level === 'warn') { - logger.warn(message, data, context); - } else { - logger.errorLog(error ?? new Error(message), message, { - context, - ...data, - }); - } - } - /** * 获取仪表盘统计数据 * 优化:使用 UNION ALL 分别查询,避免大表 JOIN,提升查询性能 @@ -363,22 +76,22 @@ export class DashboardRepository extends BaseRepository { (SELECT COUNT(*) FROM Auto_TestRun WHERE status IN ('pending', 'running')) as runningTasks `) as StatsResult[]; - const stats = this.parseStatsResult(result, { + const stats = parseStatsResult(result, { totalCases: '0', todayRuns: '0', todaySuccessRuns: '0', runningTasks: '0', }); - const totalCases = this.parseSafeInt(stats.totalCases, 0); - const todayRuns = this.parseSafeInt(stats.todayRuns, 0); - const todaySuccessRuns = this.parseSafeInt(stats.todaySuccessRuns, 0); - const runningTasks = this.parseSafeInt(stats.runningTasks, 0); + const totalCases = parseSafeInt(stats.totalCases, 0); + const todayRuns = parseSafeInt(stats.todayRuns, 0); + const todaySuccessRuns = parseSafeInt(stats.todaySuccessRuns, 0); + const runningTasks = parseSafeInt(stats.runningTasks, 0); // 成功率 = 成功次数 / 总运行次数(按运行维度计算) - const todaySuccessRate = this.calculateSuccessRate(todaySuccessRuns, todayRuns); + const todaySuccessRate = calculateSuccessRate(todaySuccessRuns, todayRuns); - this.logDashboard('debug', 'Dashboard stats retrieved', { + logDashboard('debug', 'Dashboard stats retrieved', { stats: { totalCases, todayRuns, @@ -395,7 +108,7 @@ export class DashboardRepository extends BaseRepository { runningTasks, }; } catch (error) { - this.logDashboard( + logDashboard( 'error', 'Failed to get dashboard stats', { method: 'getStats' }, @@ -445,10 +158,10 @@ export class DashboardRepository extends BaseRepository { }, LOG_CONTEXTS.DASHBOARD); return { - total: this.parseSafeInt(stats.total, 0), - passed: this.parseSafeInt(stats.passed, 0), - failed: this.parseSafeInt(stats.failed, 0), - skipped: this.parseSafeInt(stats.skipped, 0), + total: parseSafeInt(stats.total, 0), + passed: parseSafeInt(stats.passed, 0), + failed: parseSafeInt(stats.failed, 0), + skipped: parseSafeInt(stats.skipped, 0), }; } catch (error) { logger.errorLog(error, 'Failed to get today execution stats (QueryBuilder)', { @@ -500,12 +213,12 @@ export class DashboardRepository extends BaseRepository { }>; if (summaries.length > 0) { - const normalizedSummaries = this.normalizeDailySummaryRows(summaries); - if (this.hasTrendExecutionData(normalizedSummaries)) { - const result = this.buildContinuousTrendData(queryDays, normalizedSummaries); + const normalizedSummaries = normalizeDailySummaryRows(summaries); + if (hasTrendExecutionData(normalizedSummaries)) { + const result = buildContinuousTrendData(queryDays, normalizedSummaries); const duration = Date.now() - startTime; - this.logDashboard('info', 'Trend data retrieved from daily summary table', { + logDashboard('info', 'Trend data retrieved from daily summary table', { dataSource: 'summary_table', days: queryDays, recordCount: result.length, @@ -517,7 +230,7 @@ export class DashboardRepository extends BaseRepository { return result; } - this.logDashboard('warn', 'Summary table rows exist but all metrics are zero, falling back to raw tables', { + logDashboard('warn', 'Summary table rows exist but all metrics are zero, falling back to raw tables', { dataSource: 'summary_table_zero_metrics', days: queryDays, recordCount: summaries.length, @@ -548,12 +261,12 @@ export class DashboardRepository extends BaseRepository { successRate: string; }>; - const normalizedTestRunTrendData = this.normalizeDailySummaryRows(testRunTrendData); - if (this.hasTrendExecutionData(normalizedTestRunTrendData)) { - const result = this.buildContinuousTrendData(queryDays, normalizedTestRunTrendData); + const normalizedTestRunTrendData = normalizeDailySummaryRows(testRunTrendData); + if (hasTrendExecutionData(normalizedTestRunTrendData)) { + const result = buildContinuousTrendData(queryDays, normalizedTestRunTrendData); const duration = Date.now() - startTime; - this.logDashboard('info', 'Trend data calculated from Auto_TestRun', { + logDashboard('info', 'Trend data calculated from Auto_TestRun', { dataSource: 'test_run_table', days: queryDays, recordCount: result.length, @@ -566,7 +279,7 @@ export class DashboardRepository extends BaseRepository { } // 兼容旧数据:Auto_TestCaseTaskExecutions 作为最后兜底 - this.logDashboard('warn', 'Auto_TestRun has no trend data, fallback to Auto_TestCaseTaskExecutions', { + logDashboard('warn', 'Auto_TestRun has no trend data, fallback to Auto_TestCaseTaskExecutions', { dataSource: 'legacy_task_execution_fallback', days: queryDays, }); @@ -594,11 +307,11 @@ export class DashboardRepository extends BaseRepository { successRate: string; }>; - const normalizedLegacyTrendData = this.normalizeDailySummaryRows(legacyTrendData); - const finalResult = this.buildContinuousTrendData(queryDays, normalizedLegacyTrendData); + const normalizedLegacyTrendData = normalizeDailySummaryRows(legacyTrendData); + const finalResult = buildContinuousTrendData(queryDays, normalizedLegacyTrendData); const duration = Date.now() - startTime; - this.logDashboard('info', 'Trend data calculated from Auto_TestCaseTaskExecutions', { + logDashboard('info', 'Trend data calculated from Auto_TestCaseTaskExecutions', { dataSource: 'legacy_task_execution_fallback', days: queryDays, recordCount: finalResult.length, @@ -618,9 +331,9 @@ export class DashboardRepository extends BaseRepository { const maxDays = 365; const queryDays = Math.min(Math.max(days, 1), maxDays); - const dateList = Array.from(this.generateDateRange(queryDays)); - const startDate = dateList[dateList.length - 1] || this.formatLocalDate(new Date()); - const endDate = dateList[0] || this.formatLocalDate(new Date()); + const dateList = Array.from(generateDateRange(queryDays)); + const startDate = dateList[dateList.length - 1] || formatLocalDate(new Date()); + const endDate = dateList[0] || formatLocalDate(new Date()); const [summaryRows, testRunRows, taskExecutionRows] = await Promise.all([ this.dailySummaryRepository.query(` @@ -669,12 +382,12 @@ export class DashboardRepository extends BaseRepository { raw: Record ): TrendDebugSourceStats => ({ source, - rowCount: this.parseSafeInt(raw['rowCount'], 0), - daysWithData: this.parseSafeInt(raw['daysWithData'], 0), - totalExecutions: this.parseSafeInt(raw['totalExecutions'], 0), - passedCases: this.parseSafeInt(raw['passedCases'], 0), - failedCases: this.parseSafeInt(raw['failedCases'], 0), - skippedCases: this.parseSafeInt(raw['skippedCases'], 0), + rowCount: parseSafeInt(raw['rowCount'], 0), + daysWithData: parseSafeInt(raw['daysWithData'], 0), + totalExecutions: parseSafeInt(raw['totalExecutions'], 0), + passedCases: parseSafeInt(raw['passedCases'], 0), + failedCases: parseSafeInt(raw['failedCases'], 0), + skippedCases: parseSafeInt(raw['skippedCases'], 0), latestDate: raw['latestDate'] ? String(raw['latestDate']) : null, }); @@ -747,16 +460,16 @@ export class DashboardRepository extends BaseRepository { // 转换数据格式,确保所有字段都存在 return results.map((r: RecentRunRaw) => { const result: RecentRun = { - id: this.parseSafeInt(r.id, 0), + id: parseSafeInt(r.id, 0), suiteName: r.taskName || '未命名任务', status: r.status || 'pending', - duration: this.parseSafeInt(r.duration, 0), + duration: parseSafeInt(r.duration, 0), startTime: r.startTime || undefined, - totalCases: this.parseSafeInt(r.totalCases, 0), - passedCases: this.parseSafeInt(r.passedCases, 0), - failedCases: this.parseSafeInt(r.failedCases, 0), + totalCases: parseSafeInt(r.totalCases, 0), + passedCases: parseSafeInt(r.passedCases, 0), + failedCases: parseSafeInt(r.failedCases, 0), executedBy: r.executedBy || '系统', - executedById: r.executedById ? this.parseSafeInt(r.executedById) : undefined, + executedById: r.executedById ? parseSafeInt(r.executedById) : undefined, }; return result; }); @@ -870,15 +583,15 @@ export class DashboardRepository extends BaseRepository { }; // 转换为数字 - const currentRuns = this.parseSafeInt(currentData.runs, 0); - const currentPassed = this.parseSafeInt(currentData.passed, 0); - const currentFailed = this.parseSafeInt(currentData.failed, 0); - const currentTotal = this.parseSafeInt(currentData.total, 0); + const currentRuns = parseSafeInt(currentData.runs, 0); + const currentPassed = parseSafeInt(currentData.passed, 0); + const currentFailed = parseSafeInt(currentData.failed, 0); + const currentTotal = parseSafeInt(currentData.total, 0); - const previousRuns = this.parseSafeInt(previousData.runs, 0); - const previousPassed = this.parseSafeInt(previousData.passed, 0); - const previousFailed = this.parseSafeInt(previousData.failed, 0); - const previousTotal = this.parseSafeInt(previousData.total, 0); + const previousRuns = parseSafeInt(previousData.runs, 0); + const previousPassed = parseSafeInt(previousData.passed, 0); + const previousFailed = parseSafeInt(previousData.failed, 0); + const previousTotal = parseSafeInt(previousData.total, 0); // 数据验证:如果两个周期都没有数据,返回 null if (currentTotal === 0 && previousTotal === 0) { @@ -896,7 +609,7 @@ export class DashboardRepository extends BaseRepository { } // 计算环比(使用安全的百分比计算方法) - const runsComparison = this.calculatePercentage(currentRuns, previousRuns); + const runsComparison = calculatePercentage(currentRuns, previousRuns); const currentSuccessRate = currentTotal > 0 ? (currentPassed / currentTotal) * 100 : 0; const previousSuccessRate = previousTotal > 0 ? (previousPassed / previousTotal) * 100 : 0; @@ -904,7 +617,7 @@ export class DashboardRepository extends BaseRepository { ? Math.round((currentSuccessRate - previousSuccessRate) * 100) / 100 : null; - const failureComparison = this.calculatePercentage(currentFailed, previousFailed); + const failureComparison = calculatePercentage(currentFailed, previousFailed); logger.debug('Comparison data calculated', { days: queryDays, @@ -970,14 +683,14 @@ export class DashboardRepository extends BaseRepository { `) as ExecutionStats[]; // ✅ Type-safe null safety check with explicit interface - const stats = this.parseStatsResult(result, { + const stats = parseStatsResult(result, { total: '0', passed: '0', failed: '0', skipped: '0', }); - this.logDashboard('debug', 'Today execution stats retrieved', { + logDashboard('debug', 'Today execution stats retrieved', { hasData: !!result[0], resultLength: result.length, rawStats: stats, @@ -985,13 +698,13 @@ export class DashboardRepository extends BaseRepository { }); return { - total: this.parseSafeInt(stats.total, 0), - passed: this.parseSafeInt(stats.passed, 0), - failed: this.parseSafeInt(stats.failed, 0), - skipped: this.parseSafeInt(stats.skipped, 0), + total: parseSafeInt(stats.total, 0), + passed: parseSafeInt(stats.passed, 0), + failed: parseSafeInt(stats.failed, 0), + skipped: parseSafeInt(stats.skipped, 0), }; } catch (error) { - this.logDashboard( + logDashboard( 'error', 'Failed to get today execution stats', { method: 'getTodayExecution' }, @@ -1014,7 +727,7 @@ export class DashboardRepository extends BaseRepository { */ async refreshDailySummary(date?: string): Promise { try { - const targetDate = date || this.formatLocalDate(new Date()); + const targetDate = date || formatLocalDate(new Date()); // 定义查询结果接口 interface DailyStats { @@ -1053,12 +766,12 @@ export class DashboardRepository extends BaseRepository { avgDuration: '0', }; - const totalCasesRun = this.parseSafeInt(statsData.totalCasesRun, 0); - const passedCases = this.parseSafeInt(statsData.passedCases, 0); - const failedCases = this.parseSafeInt(statsData.failedCases, 0); - const skippedCases = this.parseSafeInt(statsData.skippedCases, 0); - const avgDuration = this.parseSafeInt(statsData.avgDuration, 0); - const activeCasesCount = this.parseSafeInt(activeCases[0]?.count, 0); + const totalCasesRun = parseSafeInt(statsData.totalCasesRun, 0); + const passedCases = parseSafeInt(statsData.passedCases, 0); + const failedCases = parseSafeInt(statsData.failedCases, 0); + const skippedCases = parseSafeInt(statsData.skippedCases, 0); + const avgDuration = parseSafeInt(statsData.avgDuration, 0); + const activeCasesCount = parseSafeInt(activeCases[0]?.count, 0); const successRate = totalCasesRun > 0 ? Math.round((passedCases / totalCasesRun) * 10000) / 100 @@ -1067,7 +780,7 @@ export class DashboardRepository extends BaseRepository { logger.debug('Daily summary data calculated', { targetDate, stats: { - totalExecutions: this.parseSafeInt(statsData.totalExecutions, 0), + totalExecutions: parseSafeInt(statsData.totalExecutions, 0), totalCasesRun, passedCases, failedCases, @@ -1080,7 +793,7 @@ export class DashboardRepository extends BaseRepository { await this.saveDailySummary({ summaryDate: targetDate, - totalExecutions: this.parseSafeInt(statsData.totalExecutions, 0), + totalExecutions: parseSafeInt(statsData.totalExecutions, 0), totalCasesRun, passedCases, failedCases, @@ -1181,7 +894,7 @@ export class DashboardRepository extends BaseRepository { if (datesToProcess.length === 0) { // 生成所有日期列表作为跳过的日期 const allDates: string[] = []; - for (const date of this.generateDateRange(days)) { + for (const date of generateDateRange(days)) { allDates.push(date); } @@ -1220,11 +933,11 @@ export class DashboardRepository extends BaseRepository { const activeCases = await this.testCaseRepository.query(` SELECT COUNT(*) as count FROM Auto_TestCase WHERE enabled = 1 `) as ActiveCasesStats[]; - const activeCasesCount = this.parseSafeInt(activeCases[0]?.count, 0); + const activeCasesCount = parseSafeInt(activeCases[0]?.count, 0); // 3. 构建所有日期列表(使用生成器减少内存占用) const allDates: string[] = []; - for (const date of this.generateDateRange(days)) { + for (const date of generateDateRange(days)) { allDates.push(date); } @@ -1246,12 +959,12 @@ export class DashboardRepository extends BaseRepository { // 5. 批量构建插入数据(包括没有数据的日期,填充为0) const summariesData = targetDates.map(date => { const stat = statsMap.get(date); - const totalExecutions = this.parseSafeInt(stat?.totalExecutions, 0); - const totalCasesRun = this.parseSafeInt(stat?.totalCasesRun, 0); - const passedCases = this.parseSafeInt(stat?.passedCases, 0); - const failedCases = this.parseSafeInt(stat?.failedCases, 0); - const skippedCases = this.parseSafeInt(stat?.skippedCases, 0); - const avgDuration = this.parseSafeFloat(stat?.avgDuration, 0); + const totalExecutions = parseSafeInt(stat?.totalExecutions, 0); + const totalCasesRun = parseSafeInt(stat?.totalCasesRun, 0); + const passedCases = parseSafeInt(stat?.passedCases, 0); + const failedCases = parseSafeInt(stat?.failedCases, 0); + const skippedCases = parseSafeInt(stat?.skippedCases, 0); + const avgDuration = parseSafeFloat(stat?.avgDuration, 0); const successRate = totalCasesRun > 0 ? Math.round((passedCases / totalCasesRun) * 10000) / 100 @@ -1366,4 +1079,4 @@ export class DashboardRepository extends BaseRepository { skippedDates: skippedCount > 0 ? skippedDates : undefined, }; } -} \ No newline at end of file +} diff --git a/server/repositories/DashboardRepositoryTypes.ts b/server/repositories/DashboardRepositoryTypes.ts new file mode 100644 index 0000000..f63fbf6 --- /dev/null +++ b/server/repositories/DashboardRepositoryTypes.ts @@ -0,0 +1,85 @@ +export interface DashboardStats { + totalCases: number; + todayRuns: number; + todaySuccessRate: number; + runningTasks: number; +} + +export interface TodayExecution { + total: number; + passed: number; + failed: number; + skipped: number; +} + +export interface DailySummaryData { + date: string; + totalExecutions: number; + passedCases: number; + failedCases: number; + skippedCases: number; + successRate: number; +} + +export interface RecentRun { + id: number; + suiteName?: string; + status: string; + duration: number; + startTime?: Date; + totalCases: number; + passedCases: number; + failedCases: number; + executedBy?: string; + executedById?: number; +} + +export interface TrendDebugSourceStats { + source: 'daily_summary' | 'test_run' | 'task_execution'; + rowCount: number; + daysWithData: number; + totalExecutions: number; + passedCases: number; + failedCases: number; + skippedCases: number; + latestDate: string | null; +} + +export interface TrendDebugInfo { + days: number; + dateRange: { + startDate: string; + endDate: string; + }; + sources: TrendDebugSourceStats[]; +} + +export interface ExecutionStats { + total: string; + passed: string; + failed: string; + skipped: string; +} + +export interface SummaryStats { + totalExecutions: string; + totalCasesRun: string; + passedCases: string; + failedCases: string; + skippedCases: string; + avgDuration: string; +} + +export interface ActiveCasesStats { + count: string; +} + +export interface DateStats { + summaryDate: string; + totalExecutions: string; + totalCasesRun: string; + passedCases: string; + failedCases: string; + skippedCases: string; + avgDuration: string; +} diff --git a/server/repositories/DashboardRepositoryUtils.ts b/server/repositories/DashboardRepositoryUtils.ts new file mode 100644 index 0000000..bf0f221 --- /dev/null +++ b/server/repositories/DashboardRepositoryUtils.ts @@ -0,0 +1,140 @@ +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import type { DailySummaryData } from './DashboardRepositoryTypes'; + +export function parseSafeInt( + value: string | number | null | undefined, + defaultValue: number = 0 +): number { + if (value === null || value === undefined) { + return defaultValue; + } + + const parsed = typeof value === 'string' ? parseInt(value, 10) : value; + return Number.isNaN(parsed) ? defaultValue : parsed; +} + +export function parseSafeFloat( + value: string | number | null | undefined, + defaultValue: number = 0 +): number { + if (value === null || value === undefined) { + return defaultValue; + } + + const parsed = typeof value === 'string' ? parseFloat(value) : value; + return Number.isNaN(parsed) ? defaultValue : parsed; +} + +export function calculatePercentage(current: number, previous: number): number | null { + if (previous <= 0) return null; + return Math.round(((current - previous) / previous) * 10000) / 100; +} + +export function formatLocalDate(date: Date): string { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; +} + +export function* generateDateRange(days: number): Generator { + const today = new Date(); + for (let i = 1; i <= days; i++) { + const date = new Date(today); + date.setDate(date.getDate() - i); + yield formatLocalDate(date); + } +} + +export function parseStatsResult>( + result: T[], + defaultValue: T +): T { + return result[0] || defaultValue; +} + +export function normalizeDailySummaryRows( + rows: Array<{ + date: string; + totalExecutions: string | number | null; + passedCases: string | number | null; + failedCases: string | number | null; + skippedCases: string | number | null; + successRate: string | number | null; + }> +): DailySummaryData[] { + return rows.map((row) => { + const passedCases = parseSafeInt(row.passedCases, 0); + const failedCases = parseSafeInt(row.failedCases, 0); + const skippedCases = parseSafeInt(row.skippedCases, 0); + const totalCases = passedCases + failedCases + skippedCases; + const successRate = totalCases > 0 + ? Math.round((passedCases / totalCases) * 10000) / 100 + : 0; + + return { + date: row.date, + totalExecutions: parseSafeInt(row.totalExecutions, 0), + passedCases, + failedCases, + skippedCases, + successRate, + }; + }); +} + +export function buildContinuousTrendData(days: number, rows: DailySummaryData[]): DailySummaryData[] { + const rowMap = new Map(rows.map((row) => [row.date, row])); + const continuousData: DailySummaryData[] = []; + + for (const date of generateDateRange(days)) { + continuousData.push( + rowMap.get(date) ?? { + date, + totalExecutions: 0, + passedCases: 0, + failedCases: 0, + skippedCases: 0, + successRate: 0, + } + ); + } + + return continuousData.reverse(); +} + +export function hasTrendExecutionData(rows: DailySummaryData[]): boolean { + return rows.some((row) => + row.totalExecutions > 0 || + row.passedCases > 0 || + row.failedCases > 0 || + row.skippedCases > 0 + ); +} + +export function calculateSuccessRate(passed: number, total: number): number { + if (total <= 0) return 0; + return Math.round((passed / total) * 10000) / 100; +} + +export function logDashboard( + level: 'debug' | 'info' | 'warn' | 'error', + message: string, + data?: Record, + context: string = LOG_CONTEXTS.DASHBOARD, + error?: unknown +): void { + if (level === 'debug') { + logger.debug(message, data, context); + } else if (level === 'info') { + logger.info(message, data, context); + } else if (level === 'warn') { + logger.warn(message, data, context); + } else { + logger.errorLog(error ?? new Error(message), message, { + context, + ...data, + }); + } +} diff --git a/server/repositories/ExecutionRepository.ts b/server/repositories/ExecutionRepository.ts index 18ce8a2..1b9be48 100644 --- a/server/repositories/ExecutionRepository.ts +++ b/server/repositories/ExecutionRepository.ts @@ -1,2595 +1,18 @@ -import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; -import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; -import { BaseRepository } from './BaseRepository'; -import { User } from '../entities/User'; -import logger from '../utils/logger'; -import { LOG_CONTEXTS } from '../config/logging'; -import { - TestRunStatus, - TaskExecutionStatus, - TestRunResultStatus, - TestRunResultStatusType, - TestRunTriggerTypeType, -} from '../../shared/types/execution'; - -// ============================================================================ -// 接口定义 -// ============================================================================ - -/** - * 带用户信息的 TaskExecution 接口 - */ -export interface TaskExecutionWithUser extends Omit { - executedByUser?: User; - executedByName?: string; -} - -/** - * 带用户信息的 TestRun 接口 - */ -export interface TestRunWithUser extends Omit { - triggerByUser?: User; - triggerByName?: string; -} - -/** - * 执行详情接口 - */ -export interface ExecutionDetail { - execution: TaskExecutionWithUser; - results: TestRunResult[]; -} - -/** - * 最近运行记录接口 - */ -export interface RecentExecution { - id: number; - taskId?: number; - taskName?: string; - status: string; - totalCases: number; - passedCases: number; - failedCases: number; - skippedCases: number; - duration: number; - executedBy: number; - executedByName?: string; - startTime?: Date; - endTime?: Date; - createdAt?: Date; - updatedAt?: Date; -} - -/** - * 执行结果行接口(原生 SQL 查询结果) - */ -export interface ExecutionResultRow { - id: number; - execution_id: number; - case_id: number; - case_name: string; - module: string; - priority: string; - type: string; - status: string; - start_time: Date | null; - end_time: Date | null; - duration: number | null; - error_message: string | null; - error_stack: string | null; - screenshot_path: string | null; - log_path: string | null; - assertions_total: number | null; - assertions_passed: number | null; - response_data: string | null; - created_at: Date; -} - -/** - * 测试运行行接口(原生 SQL 查询结果) - */ -export interface TestRunRow { - id: number; - project_id: number | null; - project_name: string; - status: string; - trigger_type: string; - trigger_by: number; - trigger_by_name: string; - jenkins_job: string | null; - jenkins_build_id: string | null; - jenkins_url: string | null; - abort_reason: string | null; - total_cases: number; - passed_cases: number; - failed_cases: number; - skipped_cases: number; - duration_ms: number; - start_time: Date | null; - end_time: Date | null; - created_at: Date | null; -} - -/** - * 潜在超时执行接口 - */ -export interface PotentiallyTimedOutExecution { - id: number; - jenkinsJob: string | null; - jenkinsBuildId: string | null; - startTime: Date | null; -} - -/** - * Jenkins 执行信息接口 - */ -export interface ExecutionWithJenkinsInfo { - id: number; - status: string; - jenkinsJob: string | null; - jenkinsBuildId: string | null; -} - -/** - * 卡住的执行接口 - */ -export interface StuckExecution { - id: number; - status: string; - jenkinsJob: string | null; - jenkinsBuildId: string | null; - startTime: Date | null; - durationSeconds: number; -} - -/** - * 历史卡住执行汇总 - */ -export interface StaleExecutionSummary { - stalePendingNoStartCount: number; - staleStartedCount: number; - totalStaleCount: number; - latestStalePendingCreatedAt: Date | null; -} - -/** - * 测试运行基本信息接口 - */ -export interface TestRunBasicInfo { - totalCases: number; -} - -/** - * 测试运行状态信息接口 - */ -export interface TestRunStatusInfo { - id: number; - executionId: number | null; - status: string; - jenkinsJob: string | null; - jenkinsBuildId: string | null; - jenkinsUrl: string | null; - startTime: Date | null; -} - -// ============================================================================ -// completeBatch 内部类型 -// ============================================================================ - -/** completeBatch 方法接收的单条用例结果 */ -interface BatchCaseResult { - /** caseId 可为空(如 pytest 等不携带 ID 的框架),此时通过 caseName fallback 匹配 */ - caseId?: number; - caseName: string; - status: string; - duration: number; - errorMessage?: string; - stackTrace?: string; - screenshotPath?: string; - logPath?: string; - assertionsTotal?: number; - assertionsPassed?: number; - responseData?: string; - startTime?: string | number; - endTime?: string | number; -} - -/** completeBatch 方法接收的批次结果 */ -interface BatchResults { - status: 'success' | 'failed' | 'cancelled' | 'aborted'; - passedCases: number; - failedCases: number; - skippedCases: number; - durationMs: number; - results?: BatchCaseResult[]; -} - -/** - * 运行记录 Repository - */ -export class ExecutionRepository extends BaseRepository { - // 修复2: 为私有属性添加 readonly 修饰符,防止意外重赋值 - private readonly testRunRepository: Repository; - private readonly testRunResultRepository: Repository; - private readonly testCaseRepository: Repository; - private readonly userRepository: Repository; - - // 修复8: 提取魔法数字为类静态常量,方便统一维护 - /** 批量插入的批次大小:经过性能测试,100 条/批在 MySQL 上表现最佳 */ - private static readonly BATCH_INSERT_SIZE = 100; - /** 时间窗口反查的容差(秒):允许 TestRun 与 TaskExecution 创建时间差在 ±120s 内 */ - private static readonly TIME_WINDOW_TOLERANCE_SECONDS = 120; - /** 扩大时间窗口兜底容差(秒):±300s 的宽松窗口,作为最后兜底 */ - private static readonly TIME_WINDOW_FALLBACK_SECONDS = 300; - /** getActiveRunningSlots 返回的最大记录数,避免单次查询过多 */ - private static readonly ACTIVE_SLOTS_MAX_LIMIT = 200; - - constructor(dataSource: DataSource) { - super(dataSource, TaskExecution); - this.testRunRepository = dataSource.getRepository(TestRun); - this.testRunResultRepository = dataSource.getRepository(TestRunResult); - this.testCaseRepository = dataSource.getRepository(TestCase); - this.userRepository = dataSource.getRepository(User); - } - - /** - * 创建测试运行记录 - */ - async createTestRun(runData: { - projectId: number; - triggerType: TestRunTriggerTypeType; - /** null 表示系统调度触发(无操作人) */ - triggerBy: number | null; - jenkinsJob?: string; - runConfig?: Record; - totalCases: number; - }): Promise { - const testRun = this.testRunRepository.create({ - ...runData, - status: TestRunStatus.PENDING, - }); - return this.testRunRepository.save(testRun); - } - - /** - * 创建任务运行记录 - */ - async createTaskExecution(executionData: { - taskId?: number; - taskName?: string; - totalCases: number; - /** null 表示系统调度触发(无操作人) */ - executedBy: number | null; - }): Promise { - const execution = this.repository.create({ - ...executionData, - status: TaskExecutionStatus.PENDING, - }); - return this.repository.save(execution); - } - - /** - * 批量创建测试结果记录 - * 性能优化: - * - 使用 insert 而非逐个 save,减少数据库往返 - * - 修复6: 按批次创建实体对象,避免一次性占用大量内存 - */ - async createTestResults( - results: Array<{ - executionId: number; - caseId: number; - caseName: string; - status: TestRunResultStatusType | null; - duration?: number; - errorMessage?: string; - errorStack?: string; - screenshotPath?: string; - logPath?: string; - assertionsTotal?: number; - assertionsPassed?: number; - responseData?: string; - }> - ): Promise { - if (results.length === 0) { - return; - } - - // 修复6: 按批次创建实体并插入,避免一次性将全部数据加载到内存 - for (let i = 0; i < results.length; i += ExecutionRepository.BATCH_INSERT_SIZE) { - const batch = results.slice(i, i + ExecutionRepository.BATCH_INSERT_SIZE); - const entities = batch.map(result => - this.testRunResultRepository.create(result) - ); - await this.testRunResultRepository.insert(entities); - } - } - - /** - * 更新执行状态为运行中 - * 同时清除 endTime,防止从终态回退时出现数据不一致 - */ - async markExecutionRunning(executionId: number): Promise { - await this.repository.update(executionId, { - status: TaskExecutionStatus.RUNNING, - startTime: new Date(), - endTime: null as unknown as Date, // 清除可能存在的旧 endTime,确保数据一致性 - }); - } - - /** - * 获取执行详情 - */ - async getExecutionDetail(executionId: number): Promise { - const execution = await this.repository.createQueryBuilder('execution') - .leftJoinAndSelect('execution.executedByUser', 'user') - .where('execution.id = :executionId', { executionId }) - .getOne(); - - if (!execution) { - return null; - } - - const results = await this.testRunResultRepository.find({ - where: { executionId }, - order: { id: 'ASC' }, - }); - - // 安全地获取用户名 - const executionWithUser = execution as TaskExecutionWithUser; - const executedByName = executionWithUser.executedByUser?.displayName - || executionWithUser.executedByUser?.username - || undefined; - - return { - execution: { - ...execution, - executedByName, - }, - results, - }; - } - - /** - * 执行事务 - 公开方法供Service层使用 - */ - async runInTransaction(callback: (queryRunner: QueryRunner) => Promise): Promise { - return this.executeInTransaction(callback); - } - - /** - * 获取最近运行记录 - * 修复5: 修正 getRawMany() 返回原始字段名(execution_xxx 格式),显式映射为 RecentExecution 接口字段 - */ - async getRecentExecutions(limit: number = 10): Promise { - const rawRows = await this.repository.createQueryBuilder('execution') - .leftJoin('execution.executedByUser', 'user') - .select([ - 'execution.id', - 'execution.taskId', - 'execution.taskName', - 'execution.status', - 'execution.totalCases', - 'execution.passedCases', - 'execution.failedCases', - 'execution.skippedCases', - 'execution.duration', - 'execution.executedBy', - 'execution.startTime', - 'execution.endTime', - 'user.displayName', - 'user.username', - ]) - .orderBy('execution.startTime', 'DESC') - .limit(limit) - .getRawMany(); - - // getRawMany() 返回的字段名带有 alias 前缀(如 execution_id),需要显式映射 - return rawRows.map(raw => ({ - id: raw.execution_id, - taskId: raw.execution_taskId ?? raw.execution_task_id, - taskName: raw.execution_taskName ?? raw.execution_task_name, - status: raw.execution_status, - totalCases: raw.execution_totalCases ?? raw.execution_total_cases, - passedCases: raw.execution_passedCases ?? raw.execution_passed_cases, - failedCases: raw.execution_failedCases ?? raw.execution_failed_cases, - skippedCases: raw.execution_skippedCases ?? raw.execution_skipped_cases, - duration: raw.execution_duration, - executedBy: raw.execution_executedBy ?? raw.execution_executed_by, - executedByName: raw.user_displayName ?? raw.user_display_name ?? raw.user_username ?? undefined, - startTime: raw.execution_startTime ?? raw.execution_start_time, - endTime: raw.execution_endTime ?? raw.execution_end_time, - })); - } - - /** - * 取消执行 - * 修复3: 使用枚举常量替代硬编码字符串,保持与类型系统的一致性 - */ - async cancelExecution(executionId: number): Promise { - await this.repository.update( - { id: executionId, status: In([TaskExecutionStatus.PENDING, TaskExecutionStatus.RUNNING]) }, - { - status: TaskExecutionStatus.CANCELLED, - endTime: new Date(), - } - ); - } - - /** - * 更新执行结果统计 - */ - async updateExecutionResults( - executionId: number, - results: { - status: 'success' | 'failed' | 'cancelled'; - passedCases: number; - failedCases: number; - skippedCases: number; - duration: number; - } - ): Promise { - await this.repository.update(executionId, { - ...results, - endTime: new Date(), - }); - } - - /** - * 更新测试运行结果 - */ - async updateTestRunResults( - runId: number, - results: { - status: 'success' | 'failed' | 'cancelled'; - passedCases: number; - failedCases: number; - skippedCases: number; - durationMs: number; - } - ): Promise { - await this.testRunRepository.update(runId, { - status: results.status === 'cancelled' ? 'aborted' : results.status, - passedCases: results.passedCases, - failedCases: results.failedCases, - skippedCases: results.skippedCases, - durationMs: results.durationMs, - endTime: new Date(), - }); - } - - /** - * 获取活跃用例信息 - */ - async getActiveCases(caseIds: number[]): Promise { - return this.testCaseRepository.find({ - where: { - id: In(caseIds), - enabled: true, - }, - select: ['id', 'name', 'type', 'scriptPath'], - }); - } - - /** - * 更新 Jenkins 构建信息 - */ - async updateJenkinsInfo( - runId: number, - jenkinsInfo: { - buildId: string; - buildUrl: string; - } - ): Promise { - const jobMatch = jenkinsInfo.buildUrl.match(/\/job\/([^/]+)\//); - const updateData: QueryDeepPartialEntity = { - jenkinsBuildId: jenkinsInfo.buildId, - jenkinsUrl: jenkinsInfo.buildUrl, - status: TestRunStatus.RUNNING, - startTime: new Date(), - }; - - if (jobMatch) { - updateData.jenkinsJob = jobMatch[1]; - } - - await this.testRunRepository.update(runId, updateData); - } - - /** - * 获取测试运行详情 - */ - async getTestRunDetail(runId: number): Promise { - const testRun = await this.testRunRepository.createQueryBuilder('testRun') - .leftJoinAndSelect('testRun.triggerByUser', 'user') - .where('testRun.id = :runId', { runId }) - .getOne(); - - if (!testRun) { - return null; - } - - // 安全地获取用户名 - const testRunWithUser = testRun as TestRunWithUser; - const triggerByName = testRunWithUser.triggerByUser?.displayName - || testRunWithUser.triggerByUser?.username - || undefined; - - return { - ...testRun, - triggerByName, - }; - } - - /** - * 获取测试运行详情(返回 snake_case 格式,与 TestRunRecord 接口兼容) - */ - async getTestRunDetailRow(runId: number): Promise { - const rows: TestRunRow[] = await this.testRunRepository.query(` - SELECT tr.id, tr.project_id, - CASE WHEN tr.project_id IS NOT NULL THEN CONCAT("项目 #", tr.project_id) ELSE "未分类" END as project_name, - tr.status, tr.trigger_type, tr.trigger_by, - COALESCE(u.display_name, u.username, "系统") as trigger_by_name, - tr.jenkins_job, tr.jenkins_build_id, tr.jenkins_url, - JSON_UNQUOTE(JSON_EXTRACT(tr.run_config, '$.abortReason')) AS abort_reason, - tr.total_cases, tr.passed_cases, tr.failed_cases, tr.skipped_cases, - tr.duration_ms, tr.start_time, tr.end_time, tr.created_at - FROM Auto_TestRun tr - LEFT JOIN Auto_Users u ON tr.trigger_by = u.id - WHERE tr.id = ? - `, [runId]); - return rows[0] ?? null; - } - - - /** - * 获取执行结果列表(支持分页与服务端筛选) - * @param executionId 执行ID - * @param options 分页与筛选参数 - */ - async getExecutionResults( - executionId: number, - options: { - page?: number; - pageSize?: number; - status?: string; - keyword?: string; - } = {} - ): Promise<{ data: ExecutionResultRow[]; total: number }> { - const page = Math.max(1, options.page ?? 1); - const pageSize = Math.min(100, Math.max(1, options.pageSize ?? 20)); - const offset = (page - 1) * pageSize; - - const conditions: string[] = ["r.execution_id = ?"]; - const params: (string | number)[] = [executionId]; - - if (options.status && options.status !== "all") { - // "failed" 筛选同时包含 error 状态(Jenkins 执行异常写入 error,前端统一视为失败) - if (options.status === "failed") { - conditions.push("r.status IN ('failed', 'error')"); - } else if (options.status === "pending") { - conditions.push("r.status IS NULL"); - } else { - conditions.push("r.status = ?"); - params.push(options.status); - } - } - - if (options.keyword && options.keyword.trim()) { - conditions.push("(r.case_name LIKE ? OR COALESCE(tc.module, '') LIKE ?)"); - const like = `%${options.keyword.trim()}%`; - params.push(like, like); - } - - const whereClause = `WHERE ${conditions.join(" AND ")}`; - - const data: ExecutionResultRow[] = await this.testRunResultRepository.query(` - SELECT - r.id, - r.execution_id, - r.case_id, - r.case_name, - COALESCE(tc.module, "-") as module, - COALESCE(tc.priority, "P2") as priority, - COALESCE(tc.type, "api") as type, - COALESCE(r.status, 'pending') AS status, - r.start_time, - r.end_time, - r.duration, - r.error_message, - r.error_stack, - r.screenshot_path, - r.log_path, - r.assertions_total, - r.assertions_passed, - r.response_data, - r.created_at - FROM Auto_TestRunResults r - LEFT JOIN Auto_TestCase tc ON r.case_id = tc.id - ${whereClause} - ORDER BY r.id ASC - LIMIT ? OFFSET ? - `, [...params, pageSize, offset]); - - const countResult = await this.testRunResultRepository.query(` - SELECT COUNT(*) as total - FROM Auto_TestRunResults r - LEFT JOIN Auto_TestCase tc ON r.case_id = tc.id - ${whereClause} - `, params); - const total = Number(countResult[0]?.total ?? 0); - - return { data, total }; - } - - /** - * 根据 runId 查询该批次的用例执行结果(支持分页与筛选) - * 查询策略: - * 1. 优先读 Auto_TestRun.execution_id(新数据,直接关联) - * 2. 降级到时间窗口(±120秒)+ 触发者反查 TaskExecution.id - * 3. 兜底:在 ±300秒 窗口内,取同一触发者最近的 TaskExecution,不经过中间表 - */ - async getResultsByRunId( - runId: number, - options: { page?: number; pageSize?: number; status?: string; keyword?: string; } = {} - ): Promise<{ data: ExecutionResultRow[]; total: number }> { - // 策略1:从 Auto_TestRun.execution_id 直接读(需要该字段存在且非 NULL) - let executionId: number | null = null; - try { - const testRun = await this.testRunRepository.findOne({ where: { id: runId }, select: ["executionId"] }); - executionId = testRun?.executionId ?? null; - } catch (error) { - // 修复9: 添加 debug 日志,方便排查字段不存在等问题 - logger.debug('Failed to query execution_id column from Auto_TestRun, falling back to time-window search', { - runId, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.REPOSITORY); - } - - // 策略2:降级到时间窗口(±120秒)+ 触发者反查 - if (!executionId) { - executionId = await this.findExecutionIdByRunId(runId); - } - - // 找到 executionId,走正常路径 - if (executionId) { - return this.getExecutionResults(executionId, options); - } - - // 策略3:扩大时间窗口兜底(±300秒),不要求精确匹配,取最近的 TaskExecution - logger.warn("Fallback: trying extended time-window search for runId results", { runId }, LOG_CONTEXTS.REPOSITORY); - - const runRows = await this.testRunRepository.query(` - SELECT id, trigger_by, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1 - `, [runId]) as Array<{ id: number; trigger_by: number; created_at: Date }>; - - if (!runRows || runRows.length === 0) { - logger.warn("Cannot find runId, returning empty results", { runId }, LOG_CONTEXTS.REPOSITORY); - return { data: [], total: 0 }; - } - - const runCreatedAt = runRows[0].created_at; - const triggerBy = runRows[0].trigger_by; - - // 在 ±TIME_WINDOW_FALLBACK_SECONDS 窗口内,找同一触发者最近的 TaskExecution - const fallbackRows = await this.testRunResultRepository.query(` - SELECT e.id as execution_id - FROM Auto_TestCaseTaskExecutions e - WHERE e.executed_by = ? - AND e.created_at BETWEEN DATE_SUB(?, INTERVAL ? SECOND) AND DATE_ADD(?, INTERVAL ? SECOND) - ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC - LIMIT 1 - `, [ - triggerBy, - runCreatedAt, ExecutionRepository.TIME_WINDOW_FALLBACK_SECONDS, - runCreatedAt, ExecutionRepository.TIME_WINDOW_FALLBACK_SECONDS, - runCreatedAt, - ]) as Array<{ execution_id: number }>; - - if (fallbackRows && fallbackRows.length > 0 && fallbackRows[0].execution_id) { - const fallbackExecutionId = fallbackRows[0].execution_id; - logger.info("Extended fallback found executionId", { runId, fallbackExecutionId }, LOG_CONTEXTS.REPOSITORY); - return this.getExecutionResults(fallbackExecutionId, options); - } - - logger.warn("All strategies failed to find executionId for runId, returning empty results", { runId }, LOG_CONTEXTS.REPOSITORY); - return { data: [], total: 0 }; - } - - /** - * 获取所有测试运行记录(分页 + 筛选) - * 支持按触发方式、状态、时间范围筛选 - */ - async getAllTestRuns( - limit: number = 50, - offset: number = 0, - filters: { - triggerType?: string[]; - status?: string[]; - startDate?: string; - endDate?: string; - } = {} - ): Promise<{ data: TestRunRow[]; total: number }> { - // 动态拼接 WHERE 条件 - const conditions: string[] = []; - const params: (string | number)[] = []; - - if (filters.triggerType?.length) { - const placeholders = filters.triggerType.map(() => '?').join(', '); - conditions.push('tr.trigger_type IN (' + placeholders + ')'); - params.push(...filters.triggerType); - } - - if (filters.status?.length) { - const placeholders = filters.status.map(() => '?').join(', '); - conditions.push('tr.status IN (' + placeholders + ')'); - params.push(...filters.status); - } - - if (filters.startDate) { - // 报表页展示的是触发时间,因此筛选也按 created_at 对齐 - conditions.push('tr.created_at >= ?'); - params.push(`${filters.startDate} 00:00:00`); - } - - if (filters.endDate) { - // 结束日期:当天 23:59:59(北京时间) - conditions.push('tr.created_at <= ?'); - params.push(`${filters.endDate} 23:59:59`); - } - - const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; - - // 数据查询 - const data = await this.testRunRepository.query(` - SELECT - tr.id, - tr.project_id, - CASE - WHEN tr.project_id IS NOT NULL THEN CONCAT('项目 #', tr.project_id) - ELSE '未分类' - END as project_name, - tr.status, - tr.trigger_type, - tr.trigger_by, - COALESCE(u.display_name, u.username, '系统') as trigger_by_name, - tr.jenkins_job, - tr.jenkins_build_id, - tr.jenkins_url, - JSON_UNQUOTE(JSON_EXTRACT(tr.run_config, '$.abortReason')) AS abort_reason, - tr.total_cases, - tr.passed_cases, - tr.failed_cases, - tr.skipped_cases, - tr.duration_ms, - tr.created_at, - tr.start_time, - tr.end_time - FROM Auto_TestRun tr - LEFT JOIN Auto_Users u ON tr.trigger_by = u.id - ${whereClause} - ORDER BY tr.id DESC - LIMIT ? OFFSET ? - `, [...params, limit, offset]); - - // 总数查询(同样带上筛选条件) - const countResult = await this.testRunRepository.query(` - SELECT COUNT(*) as total - FROM Auto_TestRun tr - ${whereClause} - `, params); - const total = countResult[0]?.total || 0; - - return { data, total }; - } - - /** - * 获取运行的用例列表 - * 性能优化:使用 JOIN 查询而非分步查询,消除 N+1 问题 - */ - async getRunCases(runId: number): Promise { - // 优化:使用一次 JOIN 查询获取所有数据,避免 N+1 问题 - const cases = await this.testCaseRepository - .createQueryBuilder('testCase') - .innerJoin( - 'Auto_TestRunResults', - 'result', - 'result.case_id = testCase.id AND result.execution_id = :runId', - { runId } - ) - .where('testCase.enabled = :enabled', { enabled: true }) - .select([ - 'testCase.id', - 'testCase.name', - 'testCase.type', - 'testCase.module', - 'testCase.priority', - 'testCase.scriptPath', - 'testCase.config', - ]) - .distinct(true) - .getMany(); - - return cases; - } - - /** - * 查找执行ID - * @deprecated 使用 {@link findExecutionIdByRunId} 替代,此方法只返回最新的 executionId,不够精确 - * @see findExecutionIdByRunId - */ - async findExecutionId(): Promise { - const result = await this.testRunResultRepository.createQueryBuilder('result') - .select('DISTINCT result.executionId') - .where('result.executionId IS NOT NULL') - .orderBy('result.executionId', 'DESC') - .limit(1) - .getRawOne(); - - return result?.executionId || null; - } - - /** - * 根据 runId 查找关联的 executionId - * - * 设计说明: - * - Auto_TestRun 和 Auto_TestCaseTaskExecutions 同时创建(时间相近) - * - Auto_TestRunResults 的 execution_id 指向 Auto_TestCaseTaskExecutions.id - * - 通过 Auto_TestCaseTaskExecutions 表,用触发者 + 时间窗口反查 executionId - * - * 查询策略: - * 1. 获取 TestRun 的创建时间和触发者信息 - * 2. 通过时间窗口(±TIME_WINDOW_TOLERANCE_SECONDS)+ 触发者匹配查找最近的 TaskExecution - * 3. 如果没有找到结果,记录警告并返回 null - * - * @param runId 执行批次ID - * @returns 关联的运行记录ID,如果找不到则返回 null - */ - async findExecutionIdByRunId(runId: number): Promise { - // 1. 获取 TestRun 的详细信息(含创建时间) - const testRunRows = await this.testRunRepository.query(` - SELECT id, trigger_by, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1 - `, [runId]) as Array<{ id: number; trigger_by: number; created_at: Date }>; - - if (!testRunRows || testRunRows.length === 0) { - logger.warn( - `TestRun not found`, - { runId }, - LOG_CONTEXTS.REPOSITORY - ); - return null; - } - - const testRun = testRunRows[0]; - - // 2. 通过时间窗口(±TIME_WINDOW_TOLERANCE_SECONDS)和触发者信息,在 Auto_TestCaseTaskExecutions 中查找最近的关联记录 - // 使用时间窗口而非 id 差值,避免因两表 id 自增不同步导致查找失败 - // 查询逻辑: - // - 匹配同一触发者(executed_by) - // - 创建时间在 TestRun 前后 TIME_WINDOW_TOLERANCE_SECONDS 秒内 - // - 按时间差绝对值升序排列,取最近的记录 - const result = await this.testRunResultRepository.query(` - SELECT e.id as execution_id - FROM Auto_TestCaseTaskExecutions e - WHERE e.executed_by = ? - AND e.created_at BETWEEN DATE_SUB(?, INTERVAL ? SECOND) AND DATE_ADD(?, INTERVAL ? SECOND) - ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC - LIMIT 1 - `, [ - testRun.trigger_by, - testRun.created_at, ExecutionRepository.TIME_WINDOW_TOLERANCE_SECONDS, - testRun.created_at, ExecutionRepository.TIME_WINDOW_TOLERANCE_SECONDS, - testRun.created_at, - ]); - - if (result && result.length > 0 && result[0].execution_id) { - logger.debug( - `Found executionId for runId via time-window`, - { runId, executionId: result[0].execution_id }, - LOG_CONTEXTS.REPOSITORY - ); - return result[0].execution_id; - } - - // 3. 尝试通过 TestRunResults 表反查 executionId - const fallbackExecutionId = await this.resolveExecutionIdFromRunResults(runId); - if (fallbackExecutionId) { - logger.info( - `Found executionId for runId via TestRunResults fallback`, - { runId, executionId: fallbackExecutionId }, - LOG_CONTEXTS.REPOSITORY - ); - return fallbackExecutionId; - } - - // 4. 如果没有找到,记录警告 - logger.warn( - `Could not find executionId for runId`, - { - runId, - triggerBy: testRun.trigger_by, - suggestion: 'Consider adding execution_id column to Auto_TestRun table' - }, - LOG_CONTEXTS.REPOSITORY - ); - - return null; - } - - /** - * 更新测试结果 - * 修复4: 使用 TestRunResultStatus 枚举校验状态,替代不安全的强制类型断言 - */ - async updateTestResult( - executionId: number, - caseId: number | undefined | null, - result: { - status: string; - duration: number; - errorMessage?: string; - errorStack?: string; - screenshotPath?: string; - logPath?: string; - assertionsTotal?: number; - assertionsPassed?: number; - responseData?: string; - startTime?: Date; - endTime?: Date; - caseName?: string; - } - ): Promise { - // 将外部传入的 status 字符串映射为合法的枚举值,不合法时降级为 'error' - const validStatuses: ReadonlyArray = Object.values(TestRunResultStatus); - const safeStatus = validStatuses.includes(result.status) - ? (result.status as 'passed' | 'failed' | 'skipped' | 'error') - : 'error'; - - const updateData: Partial = { - status: safeStatus, - duration: result.duration, - errorMessage: result.errorMessage || null, - errorStack: result.errorStack || null, - screenshotPath: result.screenshotPath || null, - logPath: result.logPath || null, - assertionsTotal: result.assertionsTotal || null, - assertionsPassed: result.assertionsPassed || null, - responseData: result.responseData || null, - startTime: result.startTime || null, - endTime: result.endTime || null, - }; - - // 【修复】优先用 caseId 匹配;无 caseId 或为 0 时降级用 caseName 匹配(Jenkins 只传 caseName 的场景) - if (caseId && caseId > 0) { - const updateResult = await this.testRunResultRepository.update( - { executionId, caseId }, - updateData - ); - if ((updateResult.affected ?? 0) > 0) return true; - } - - // 【修复】当 caseId 缺失或为 0 时,尝试用 caseName 匹配 - // 优先精确匹配,其次模糊匹配(以防格式略有差异) - if (result.caseName) { - // 【第 2 层】精确匹配(完全相同) - const updateResult = await this.testRunResultRepository - .createQueryBuilder() - .update(TestRunResult) - .set(updateData) - .where('execution_id = :executionId AND case_name = :caseName', { - executionId, - caseName: result.caseName, - }) - .execute(); - if ((updateResult.affected ?? 0) > 0) return true; - - // 【第 3 层】大小写不敏感的精确匹配 - const caseInsensitiveResult = await this.testRunResultRepository - .createQueryBuilder() - .update(TestRunResult) - .set(updateData) - .where('execution_id = :executionId AND LOWER(case_name) = LOWER(:caseName)', { - executionId, - caseName: result.caseName, - }) - .execute(); - if ((caseInsensitiveResult.affected ?? 0) > 0) return true; - - // 【第 4 层】包含式模糊匹配(占位符 caseName 包含回调的 caseName) - // 例:期望 'TestGeolocation::test_geolocation' 但收到 'test_geolocation' - const fuzzyUpdateResult = await this.testRunResultRepository - .createQueryBuilder() - .update(TestRunResult) - .set(updateData) - .where('execution_id = :executionId AND (LOWER(case_name) LIKE LOWER(:fuzzyPattern1) OR LOWER(case_name) LIKE LOWER(:fuzzyPattern2))', { - executionId, - fuzzyPattern1: `%${result.caseName}%`, - fuzzyPattern2: `%${result.caseName.replace(/.*::/g, '')}%`, // 去掉命名空间 - }) - .execute(); - if ((fuzzyUpdateResult.affected ?? 0) > 0) return true; - } - - return false; - } - - /** - * 创建测试结果记录 - * 修复4: 使用 TestRunResultStatus 枚举校验状态,替代不安全的强制类型断言 - */ - async createTestResult(result: { - executionId: number; - caseId: number; - caseName: string; - status: string; - duration: number; - errorMessage?: string; - errorStack?: string; - screenshotPath?: string; - logPath?: string; - assertionsTotal?: number; - assertionsPassed?: number; - responseData?: string; - startTime?: Date; - endTime?: Date; - }): Promise { - // 将外部传入的 status 字符串映射为合法的枚举值,不合法时降级为 'error' - const validStatuses: ReadonlyArray = Object.values(TestRunResultStatus); - const safeStatus = validStatuses.includes(result.status) - ? (result.status as 'passed' | 'failed' | 'skipped' | 'error') - : 'error'; - - const entity = this.testRunResultRepository.create({ - executionId: result.executionId, - caseId: result.caseId, - caseName: result.caseName, - status: safeStatus, - duration: result.duration, - errorMessage: result.errorMessage || null, - errorStack: result.errorStack || null, - screenshotPath: result.screenshotPath || null, - logPath: result.logPath || null, - assertionsTotal: result.assertionsTotal || null, - assertionsPassed: result.assertionsPassed || null, - responseData: result.responseData || null, - startTime: result.startTime || null, - endTime: result.endTime || null, - }); - await this.testRunResultRepository.save(entity); - } - - /** - * 标记执行为超时 - */ - async markExecutionAsTimedOut(runId: number): Promise { - await this.testRunRepository.update(runId, { - status: TestRunStatus.ABORTED, - endTime: new Date(), - }); - } - - /** - * 获取可能超时的运行记录 - */ - async getPotentiallyTimedOutExecutions(timeoutThreshold: Date): Promise { - return this.testRunRepository.createQueryBuilder('testRun') - .select([ - 'testRun.id', - 'testRun.jenkinsJob', - 'testRun.jenkinsBuildId', - 'testRun.startTime', - ]) - .where('testRun.status IN (:...statuses)', { statuses: [TestRunStatus.PENDING, TestRunStatus.RUNNING] }) - .andWhere('testRun.startTime < :timeoutThreshold', { timeoutThreshold }) - .getRawMany(); - } - - /** - * 获取有 Jenkins 信息的运行记录 - */ - async getExecutionsWithJenkinsInfo(limit: number = 50): Promise { - return this.testRunRepository.createQueryBuilder('testRun') - .select([ - 'testRun.id', - 'testRun.status', - 'testRun.jenkinsJob', - 'testRun.jenkinsBuildId', - ]) - .where('testRun.jenkinsJob IS NOT NULL') - .andWhere('testRun.jenkinsBuildId IS NOT NULL') - .orderBy('testRun.id', 'DESC') - .limit(limit) - .getRawMany(); - } - - /** - * 获取可能卡住的运行记录(用于 ExecutionMonitorService) - * 查询状态为 pending/running 且超过指定时间阈值的运行记录 - */ - async getPotentiallyStuckExecutions(thresholdSeconds: number, limit: number = 20): Promise { - // 只检查最近 N 小时内的执行(优化:避免查询过期的旧执行) - // 从环境变量读取配置,默认 24 小时 - const maxAgeHours = parseInt(process.env.EXECUTION_MONITOR_MAX_AGE_HOURS || '24', 10); - - return this.testRunRepository.createQueryBuilder('testRun') - .select([ - 'testRun.id as id', - 'testRun.status as status', - 'testRun.jenkinsJob as jenkinsJob', - 'testRun.jenkinsBuildId as jenkinsBuildId', - 'testRun.startTime as startTime', - 'TIMESTAMPDIFF(SECOND, testRun.startTime, NOW()) as durationSeconds', - ]) - .where('testRun.status IN (:...statuses)', { statuses: [TestRunStatus.PENDING, TestRunStatus.RUNNING] }) - .andWhere('testRun.startTime IS NOT NULL') - .andWhere('TIMESTAMPDIFF(SECOND, testRun.startTime, NOW()) > :thresholdSeconds', { thresholdSeconds }) - // 只检查最近 N 小时内启动的执行(避免查询过期执行,用 start_time 代替 created_at) - .andWhere('testRun.startTime > DATE_SUB(NOW(), INTERVAL :maxAgeHours HOUR)', { maxAgeHours }) - .orderBy('testRun.startTime', 'ASC') - .limit(limit) - .getRawMany(); - } - - /** - * 获取测试运行的基本信息 - */ - async getTestRunBasicInfo(runId: number): Promise { - return this.testRunRepository.findOne({ - where: { id: runId }, - select: ['totalCases'], - }); - } - - /** - * 标记超时的旧执行为 aborted(清理过期卡住的执行) - * 覆盖两类僵尸记录: - * 1. start_time 不为 null 且超过 maxAgeHours(正常超时的 running/pending) - * 2. start_time IS NULL 且 created_at 超过 stuckPendingMinutes(Jenkins 从未触发的 pending,服务重启时队列丢失) - * @param maxAgeHours 最大运行时长(小时),超过时清理 start_time 不为 null 的记录 - * @param stuckPendingMinutes Jenkins 未触发的 pending 最长保留时间(分钟),默认 10 分钟 - * @returns 更新的执行数量 - */ - async markOldStuckExecutionsAsAbandoned(maxAgeHours: number = 24, stuckPendingMinutes: number = 10): Promise { - // 使用原生 SQL 支持 OR 条件,避免 TypeORM QueryBuilder 的 OR 限制 - const result = await this.testRunRepository.query( - `UPDATE Auto_TestRun - SET status = 'aborted', end_time = NOW() - WHERE status IN ('pending', 'running') - AND ( - -- 类型1:已开始但运行超时(start_time 不为 null) - (start_time IS NOT NULL AND start_time < DATE_SUB(NOW(), INTERVAL ? HOUR)) - OR - -- 类型2:Jenkins 从未触发(start_time 为 null),创建超过 N 分钟的 pending 记录 - (start_time IS NULL AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE)) - )`, - [maxAgeHours, stuckPendingMinutes] - ) as { affectedRows?: number; changedRows?: number }; - - return result?.affectedRows ?? 0; - } - - /** - * 汇总历史卡住记录(用于运行记录页提示条) - */ - async getStaleExecutionSummary(maxAgeHours: number = 24, stuckPendingMinutes: number = 10): Promise { - const rows = await this.testRunRepository.query( - `SELECT - SUM( - CASE - WHEN status = 'pending' - AND start_time IS NULL - AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE) - THEN 1 ELSE 0 - END - ) AS stale_pending_no_start_count, - SUM( - CASE - WHEN status IN ('pending', 'running') - AND start_time IS NOT NULL - AND start_time < DATE_SUB(NOW(), INTERVAL ? HOUR) - THEN 1 ELSE 0 - END - ) AS stale_started_count, - MAX( - CASE - WHEN status = 'pending' - AND start_time IS NULL - AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE) - THEN created_at ELSE NULL - END - ) AS latest_stale_pending_created_at - FROM Auto_TestRun - WHERE status IN ('pending', 'running')`, - [stuckPendingMinutes, maxAgeHours, stuckPendingMinutes] - ) as Array<{ - stale_pending_no_start_count: number | string | null; - stale_started_count: number | string | null; - latest_stale_pending_created_at: Date | string | null; - }>; - - const row = rows[0] ?? { - stale_pending_no_start_count: 0, - stale_started_count: 0, - latest_stale_pending_created_at: null, - }; - - const stalePendingNoStartCount = Number(row.stale_pending_no_start_count ?? 0); - const staleStartedCount = Number(row.stale_started_count ?? 0); - - return { - stalePendingNoStartCount, - staleStartedCount, - totalStaleCount: stalePendingNoStartCount + staleStartedCount, - latestStalePendingCreatedAt: row.latest_stale_pending_created_at ? new Date(row.latest_stale_pending_created_at) : null, - }; - } - - /** - * 完整的触发测试执行流程(包含事务) - */ - async triggerExecution(input: { - caseIds: number[]; - projectId: number; - /** null 表示系统调度触发(无操作人) */ - triggeredBy: number | null; - triggerType: 'manual' | 'jenkins' | 'schedule'; - jenkinsJob?: string; - runConfig?: Record; - taskId?: number; - taskName?: string; - }): Promise<{ runId: number; executionId: number; totalCases: number; caseIds: number[] }> { - return this.executeInTransaction(async (_queryRunner) => { - // 1. 获取活跃用例 - const cases = await this.testCaseRepository.find({ - where: { - id: In(input.caseIds), - enabled: true, - }, - select: ['id', 'name', 'type', 'scriptPath'], - }); - - if (!cases || cases.length === 0) { - throw new Error(`No active test cases found with IDs: ${input.caseIds.join(',')}`); - } - - // 2. 创建测试运行记录 - const testRun = await this.createTestRun({ - projectId: input.projectId, - triggerType: input.triggerType, - triggerBy: input.triggeredBy, - jenkinsJob: input.jenkinsJob, - runConfig: input.runConfig, - totalCases: cases.length, - }); - - // 3. 创建任务运行记录 - const taskExecution = await this.createTaskExecution({ - taskId: input.taskId, - taskName: input.taskName, - totalCases: cases.length, - executedBy: input.triggeredBy, - }); - - // 4. 回填 executionId 到 TestRun(直接关联,消除时间窗口反查依赖) - await this.testRunRepository.update(testRun.id, { executionId: taskExecution.id }); - - // 5. 批量创建测试结果记录(初始状态为 error,等待 Jenkins 回调更新) - const testResults = cases.map(testCase => ({ - executionId: taskExecution.id, - caseId: testCase.id, - caseName: testCase.name, - status: null, - })); - - await this.createTestResults(testResults); - - return { - runId: testRun.id, - executionId: taskExecution.id, - totalCases: cases.length, - caseIds: cases.map(c => c.id), - }; - }); - } - - /** - * 完成批次执行 - * - * 修复10: 将原来 281 行的大方法拆分为若干职责明确的私有方法: - * - resolveExecutionIdForBatch: 多策略解析 executionId - * - syncTaskExecutionStatus: 同步 TaskExecution 状态 - * - updateDetailedCaseResults: 处理带详细结果的更新路径 - * - updateSummaryOnlyResults: 处理只有汇总统计的兜底路径 - * - * @param runId 执行批次ID - * @param results 执行结果 - * @param executionId 可选的执行ID(来自缓存,用于优化) - */ - async completeBatch( - runId: number, - results: BatchResults, - executionId?: number - ): Promise { - // 0. 二次校验:防止并发回调导致正确结果被错误数据覆盖 - // 注意:这里不能使用 FOR UPDATE + 事务外 Repository 混用, - // 否则会出现“当前事务持锁、另一个连接更新同一行”导致 lock wait timeout。 - // 改为轻量读取 + 幂等防回退,避免锁竞争。 - const lockResult = await this.testRunRepository.query(` - SELECT id, status, passed_cases, failed_cases, skipped_cases - FROM Auto_TestRun - WHERE id = ? - LIMIT 1 - `, [runId]) as Array<{ - id: number; - status: string; - passed_cases: number | null; - failed_cases: number | null; - skipped_cases: number | null; - }>; - - const currentTestRun = lockResult.length > 0 ? { - id: lockResult[0].id, - status: lockResult[0].status, - passedCases: lockResult[0].passed_cases, - failedCases: lockResult[0].failed_cases, - skippedCases: lockResult[0].skipped_cases, - } : null; - - const finalStatuses = ['success', 'failed', 'cancelled', 'aborted']; - const hasDetailedResults = Array.isArray(results.results) && results.results.length > 0; - const hasSummaryCounts = (results.passedCases + results.failedCases + results.skippedCases) > 0; - - if (currentTestRun && finalStatuses.includes(currentTestRun.status)) { - // 数据版本检查:判断新数据是否比现有数据"更好" - const currentHasRealData = (currentTestRun.passedCases ?? 0) > 0 || - (currentTestRun.failedCases ?? 0) > 0 || - (currentTestRun.skippedCases ?? 0) > 0; - - // 终态已存在真实数据时,检查新数据是否会导致数据"倒退" - // 修复:无论新回调是否有详细结果,都要检查数据质量 - if (currentHasRealData) { - const currentTotal = (currentTestRun.passedCases ?? 0) + - (currentTestRun.failedCases ?? 0) + - (currentTestRun.skippedCases ?? 0); - const newTotal = results.passedCases + results.failedCases + results.skippedCases; - const currentPassed = currentTestRun.passedCases ?? 0; - const newPassed = results.passedCases; - - // 判断数据是否变差: - // 1. 总量减少(测试结果不完整) - // 2. 总量相同但 passed 减少(正确结果被错误结果覆盖) - // 3. 无详细结果且回调数据总量相同或更少(可能是空回调或部分数据) - const isDataRegression = newTotal < currentTotal || - (newTotal === currentTotal && newPassed < currentPassed) || - (!hasDetailedResults && newTotal <= currentTotal); - - if (isDataRegression) { - logger.warn( - 'completeBatch: rejected regression update - existing data is better than new callback', - { - runId, - currentStatus: currentTestRun.status, - currentPassed: currentTestRun.passedCases, - currentFailed: currentTestRun.failedCases, - currentSkipped: currentTestRun.skippedCases, - newPassed: results.passedCases, - newFailed: results.failedCases, - newSkipped: results.skippedCases, - hasDetailedResults, - newTotal, - currentTotal, - source: 'concurrent_callback_protection' - }, - LOG_CONTEXTS.REPOSITORY - ); - return; // 拒绝更新,保留现有的正确数据 - } - } - - logger.info( - 'completeBatch: allowing update to completed run with new payload', - { - runId, - currentStatus: currentTestRun.status, - hasDetailedResults, - hasSummaryCounts, - source: 'concurrent_callback_protection' - }, - LOG_CONTEXTS.REPOSITORY - ); - } - - // 1. 更新 TestRun 记录(将 cancelled 映射为 aborted 以兼容数据库枚举) - const mappedStatus = this.mapStatusForTestRun(results.status); - await this.testRunRepository.update(runId, { - status: mappedStatus, - passedCases: results.passedCases, - failedCases: results.failedCases, - skippedCases: results.skippedCases, - durationMs: results.durationMs, - endTime: new Date(), - }); - - // 2. 多策略解析 executionId - const resolvedExecutionId = await this.resolveExecutionIdForBatch(runId, executionId); - - // 【诊断日志】记录 executionId 解析结果,便于追踪「汇总 passed 但明细 FAILED」问题 - logger.info( - 'completeBatch: executionId resolution result', - { - runId, - cachedExecutionId: executionId, - resolvedExecutionId, - hasResults: !!(results.results && results.results.length > 0), - resultsCount: results.results?.length ?? 0, - summaryPassed: results.passedCases, - summaryFailed: results.failedCases, - summarySkipped: results.skippedCases, - }, - LOG_CONTEXTS.REPOSITORY - ); - - // 3. 同步 TaskExecution 状态(与 TestRun 保持一致) - if (resolvedExecutionId) { - await this.syncTaskExecutionStatus(resolvedExecutionId, results); - } - - // 4. 更新详细用例结果 - if (resolvedExecutionId) { - if (results.results && results.results.length > 0) { - await this.updateDetailedCaseResults(runId, resolvedExecutionId, results.results); - - // 修复12: 清理残留的 ERROR 占位符 - // 当 Jenkins 回调只返回部分测试结果时(如10个用例只返回3个), - // updateDetailedCaseResults 只会更新这3个用例,剩余的 ERROR 占位符不会被清理。 - // 需要根据整体执行状态批量更新这些残留占位符。 - await this.cleanupResidualErrorPlaceholders( - runId, - resolvedExecutionId, - results.results.length, - results - ); - } else { - await this.updateSummaryOnlyResults(runId, resolvedExecutionId, results); - } - } else { - logger.warn( - `Could not determine executionId for runId, skipping detailed result updates`, - { runId, resultsCount: results.results?.length ?? 0, cachedExecutionId: executionId }, - LOG_CONTEXTS.REPOSITORY - ); - - // 【兜底修复】resolvedExecutionId 无法从 TaskExecution 表获取时, - // 尝试直接从 Auto_TestRunResults 表通过已有占位记录反查 executionId, - // 确保 ERROR 占位符依然能被清理,避免用例状态永久卡在 error - const fallbackExecId = await this.resolveExecutionIdFromRunResults(runId); - logger.info( - `completeBatch: fallback executionId lookup result`, - { runId, fallbackExecId: fallbackExecId ?? null, found: !!fallbackExecId }, - LOG_CONTEXTS.REPOSITORY - ); - if (fallbackExecId) { - await this.updateSummaryOnlyResults(runId, fallbackExecId, results); - await this.performFinalErrorCleanup(fallbackExecId, results); - // 【修复3】兜底路径也要做最终一致性校正,确保 Auto_TestRun 统计与明细一致 - await this.reconcileBatchSummary(runId, fallbackExecId, mappedStatus); - } - } - - // 【安全防护】无论采用哪个更新路径,都执行最后的全局清理 - // 防止在特殊场景(既无详细结果也无汇总统计)下 ERROR 占位符残留 - if (resolvedExecutionId) { - await this.performFinalErrorCleanup(resolvedExecutionId, results); - // 最终一致性校正:以结果表为准回填批次汇总,避免"汇总 success 但明细仍 error" - await this.reconcileBatchSummary(runId, resolvedExecutionId, mappedStatus); - } - } - - /** - * 最终一致性校正:以 Auto_TestRunResults 结果表为准,回填 Auto_TestRun 和 Auto_TestCaseTaskExecutions 的汇总统计 - * 确保两张表数据一致,避免"汇总通过但明细失败"或相反的情况 - */ - private async reconcileBatchSummary( - runId: number, - executionId: number, - mappedStatus: 'pending' | 'running' | 'success' | 'failed' | 'aborted' - ): Promise { - const finalCounts = await this.countResultsByStatus(executionId); - if (finalCounts.total > 0) { - let reconciledRunStatus = mappedStatus; - if (finalCounts.failed > 0) { - reconciledRunStatus = 'failed'; - } else if (finalCounts.passed > 0) { - reconciledRunStatus = 'success'; - } - - await this.testRunRepository.update(runId, { - status: reconciledRunStatus, - passedCases: finalCounts.passed, - failedCases: finalCounts.failed, - skippedCases: finalCounts.skipped, - }); - - const reconciledTaskStatus: 'success' | 'failed' | 'cancelled' = - reconciledRunStatus === 'aborted' ? 'cancelled' - : reconciledRunStatus === 'success' ? 'success' - : 'failed'; - - await this.repository.update(executionId, { - status: reconciledTaskStatus, - passedCases: finalCounts.passed, - failedCases: finalCounts.failed, - skippedCases: finalCounts.skipped, - }); - - logger.info('Reconciled batch summary from result rows', { - runId, - executionId, - reconciledRunStatus, - finalCounts, - }, LOG_CONTEXTS.REPOSITORY); - } - } - - /** - * 【安全防护】最终的全局 ERROR 清理 - * 在所有其他更新完成后执行,确保不会有遗漏的 ERROR 占位符 - * 这是一个防御性的清理,不依赖任何前置条件 - */ - private async performFinalErrorCleanup(executionId: number, results: BatchResults): Promise { - try { - const errorRows = await this.testRunResultRepository.query(` - SELECT COUNT(*) AS errorCount - FROM Auto_TestRunResults - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - `, [executionId]) as Array<{ errorCount: string }>; - - const residualErrorCount = Number(errorRows[0]?.errorCount ?? 0); - - if (residualErrorCount === 0) { - logger.debug( - 'Final error cleanup: no orphaned errors found', - { executionId }, - LOG_CONTEXTS.REPOSITORY - ); - return; - } - - // 根据运行状态决定清理目标 - let targetStatus: 'passed' | 'failed' | 'skipped'; - const mappedStatus = this.mapStatusForTestRun(results.status); - - if (mappedStatus === 'success') { - targetStatus = 'passed'; - } else if (mappedStatus === 'failed') { - targetStatus = 'failed'; - } else { - targetStatus = 'skipped'; - } - - const cleaned = await this.bulkUpdateErrorResults(executionId, targetStatus); - - logger.info( - 'Final error cleanup completed', - { - executionId, - residualErrorCount, - cleaned, - targetStatus, - reason: 'safety-cleanup-after-all-updates', - }, - LOG_CONTEXTS.REPOSITORY - ); - } catch (error) { - logger.warn( - 'Final error cleanup failed, but execution will continue', - { - executionId, - error: error instanceof Error ? error.message : String(error), - }, - LOG_CONTEXTS.REPOSITORY - ); - } - } - - /** - * 【轮询路径清理】清理指定 execution 中的 ERROR 占位符 - * 用于轮询同步路径中,当 Jenkins 状态已完成但结果为虚拟/部分时,清理预创建的 ERROR 占位符 - */ - async cleanupErrorPlaceholdersForExecution(executionId: number, runStatus: string): Promise { - try { - // 查询是否存在 ERROR 占位符 - const errorRows = await this.testRunResultRepository.query(` - SELECT COUNT(*) AS errorCount - FROM Auto_TestRunResults - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - `, [executionId]) as Array<{ errorCount: string }>; - - const residualErrorCount = Number(errorRows[0]?.errorCount ?? 0); - - if (residualErrorCount === 0) { - logger.debug( - 'cleanupErrorPlaceholdersForExecution: no errors found', - { executionId }, - LOG_CONTEXTS.REPOSITORY - ); - return 0; - } - - // 根据运行状态映射清理目标 - let targetStatus: 'passed' | 'failed' | 'skipped'; - if (runStatus === 'success') { - targetStatus = 'passed'; - } else if (runStatus === 'failed') { - targetStatus = 'failed'; - } else { - targetStatus = 'skipped'; - } - - const cleaned = await this.bulkUpdateErrorResults(executionId, targetStatus); - - logger.info( - 'Cleaned error placeholders in polling sync path', - { - executionId, - runStatus, - residualErrorCount, - cleaned, - targetStatus, - reason: 'polling-sync-cleanup', - }, - LOG_CONTEXTS.REPOSITORY - ); - - return cleaned; - } catch (error) { - logger.warn( - 'Failed to cleanup error placeholders', - { - executionId, - error: error instanceof Error ? error.message : String(error), - }, - LOG_CONTEXTS.REPOSITORY - ); - return 0; - } - } - - /** - * 【紧急修复】修复孤立的 TestRun(没有绑定 execution_id) - * - * 问题:旧的 TestRun 没有设置 execution_id 字段,导致查询结果时通过时间窗口反查 - * 可能得到错误的 executionId,进而获取错误的用例结果 - * - * 解决方案: - * 1. 查找所有 execution_id 为 NULL 的 TestRun - * 2. 通过触发者 + 时间窗口匹配最近的 TaskExecution - * 3. 回填 execution_id 字段 - */ - async fixOrphanedTestRuns(): Promise<{ fixed: number; checked: number }> { - let fixed = 0; - let checked = 0; - - try { - // 查找所有 execution_id 为 NULL 的 TestRun - const orphanedRuns = await this.testRunRepository.query(` - SELECT tr.id, tr.trigger_by, tr.created_at - FROM Auto_TestRun tr - WHERE tr.execution_id IS NULL - ORDER BY tr.id DESC - LIMIT 100 - `) as Array<{ id: number; trigger_by: number; created_at: Date }>; - - checked = orphanedRuns.length; - - if (orphanedRuns.length === 0) { - logger.info('No orphaned TestRuns found', {}, LOG_CONTEXTS.REPOSITORY); - return { fixed: 0, checked: 0 }; - } - - logger.info(`Found ${orphanedRuns.length} orphaned TestRuns, attempting to fix...`, {}, LOG_CONTEXTS.REPOSITORY); - - for (const run of orphanedRuns) { - // 通过时间窗口 + 触发者查找最近的 TaskExecution - const matchingExecutions = await this.repository.query(` - SELECT e.id as execution_id - FROM Auto_TestCaseTaskExecutions e - WHERE e.executed_by = ? - AND e.created_at BETWEEN DATE_SUB(?, INTERVAL 120 SECOND) AND DATE_ADD(?, INTERVAL 120 SECOND) - ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC - LIMIT 1 - `, [ - run.trigger_by, - run.created_at, run.created_at, run.created_at - ]) as Array<{ execution_id: number }>; - - if (matchingExecutions && matchingExecutions.length > 0) { - const executionId = matchingExecutions[0].execution_id; - await this.testRunRepository.update(run.id, { executionId }); - fixed++; - logger.info(`Fixed TestRun #${run.id} → executionId ${executionId}`, {}, LOG_CONTEXTS.REPOSITORY); - } - } - - logger.info(`Orphaned TestRuns fix completed: ${fixed}/${checked} fixed`, { fixed, checked }, LOG_CONTEXTS.REPOSITORY); - } catch (error) { - logger.warn( - 'Failed to fix orphaned TestRuns', - { error: error instanceof Error ? error.message : String(error) }, - LOG_CONTEXTS.REPOSITORY - ); - } - - return { fixed, checked }; - } - - /** - * 更新测试运行的状态和 Jenkins 信息(用于 Jenkins 同步) - */ - async updateTestRunStatus( - runId: number, - status: string, - options?: { - durationMs?: number; - passedCases?: number; - failedCases?: number; - skippedCases?: number; - abortReason?: string; - } - ): Promise { - const normalizedStatus = status === 'cancelled' ? 'aborted' : status; - const updateData: QueryDeepPartialEntity = { - status: normalizedStatus as 'pending' | 'running' | 'success' | 'failed' | 'aborted', - }; - - // 终态时设置 endTime - if (['success', 'failed', 'aborted'].includes(normalizedStatus)) { - updateData.endTime = new Date(); - } else if (['running', 'pending'].includes(normalizedStatus)) { - // 非终态(running/pending)时清除 endTime,防止从终态回退时出现数据不一致 - // 场景:Jenkins 轮询误判(building=true)或网络延迟导致状态错乱 - // 确保数据一致性:running/pending 状态不应有结束时间 - updateData.endTime = null as unknown as Date; - } - - if (options?.durationMs !== undefined) updateData.durationMs = options.durationMs; - if (options?.passedCases !== undefined) updateData.passedCases = options.passedCases; - if (options?.failedCases !== undefined) updateData.failedCases = options.failedCases; - if (options?.skippedCases !== undefined) updateData.skippedCases = options.skippedCases; - - if (normalizedStatus === 'aborted' && options?.abortReason) { - const current = await this.testRunRepository.findOne({ - where: { id: runId }, - select: ['id', 'runConfig'], - }); - const currentConfig = current?.runConfig; - let parsedConfig: Record = {}; - - if (currentConfig && typeof currentConfig === 'object' && !Array.isArray(currentConfig)) { - parsedConfig = currentConfig as Record; - } else if (typeof currentConfig === 'string') { - try { - const parsed = JSON.parse(currentConfig) as unknown; - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - parsedConfig = parsed as Record; - } - } catch { - parsedConfig = {}; - } - } - - updateData.runConfig = { - ...parsedConfig, - abortReason: options.abortReason, - abortAt: new Date().toISOString(), - }; - } - - await this.testRunRepository.update(runId, updateData); - } - - /** - * 获取测试运行状态信息 - */ - async getTestRunStatus(runId: number): Promise { - return this.testRunRepository.findOne({ - where: { id: runId }, - select: ['id', 'executionId', 'status', 'jenkinsJob', 'jenkinsBuildId', 'jenkinsUrl', 'startTime'], - }); - } - - async syncTaskExecutionFromTestRunStatus( - runId: number, - status: string, - options?: { - durationMs?: number; - passedCases?: number; - failedCases?: number; - skippedCases?: number; - } - ): Promise { - const testRun = await this.testRunRepository.findOne({ - where: { id: runId }, - select: ['id', 'executionId', 'passedCases', 'failedCases', 'skippedCases', 'durationMs'], - }); - - if (!testRun?.executionId) { - logger.warn('syncTaskExecutionFromTestRunStatus: no executionId bound to run', { - runId, - }, LOG_CONTEXTS.REPOSITORY); - return; - } - - const currentTaskExecution = await this.repository.findOne({ - where: { id: testRun.executionId }, - select: ['id', 'startTime', 'endTime'], - }); - - const normalizedStatus: 'pending' | 'running' | 'success' | 'failed' | 'cancelled' = - status === 'aborted' || status === 'cancelled' ? 'cancelled' - : status === 'success' ? 'success' - : status === 'running' ? 'running' - : status === 'pending' ? 'pending' - : 'failed'; - - const updateData: QueryDeepPartialEntity = { - status: normalizedStatus, - passedCases: options?.passedCases ?? testRun.passedCases ?? 0, - failedCases: options?.failedCases ?? testRun.failedCases ?? 0, - skippedCases: options?.skippedCases ?? testRun.skippedCases ?? 0, - duration: Math.round((options?.durationMs ?? testRun.durationMs ?? 0) / 1000), - }; - - if (normalizedStatus === 'running') { - updateData.startTime = currentTaskExecution?.startTime ?? new Date(); - updateData.endTime = null as unknown as Date; - } else if (['success', 'failed', 'cancelled'].includes(normalizedStatus)) { - updateData.startTime = currentTaskExecution?.startTime ?? new Date(); - updateData.endTime = currentTaskExecution?.endTime ?? new Date(); - } - - await this.repository.update(testRun.executionId, updateData); - } - - /** - * [dev-11] 获取当前所有 running 状态的执行记录(用于服务启动时恢复调度器槽位) - * 只查最近 maxAgeHours 小时内启动的执行,避免捞出陈年旧账 - * 使用原生 SQL 避免 TypeORM 实体元数据依赖(防止启动顺序竞态问题) - * 修复12: 添加 ACTIVE_SLOTS_MAX_LIMIT 上限,防止异常场景下返回过多记录 - */ - async getActiveRunningSlots(maxAgeHours: number = 24): Promise> { - const rows = await this.testRunRepository.query( - `SELECT r.id, e.task_id AS taskId, r.start_time AS startTime - FROM Auto_TestRun r - LEFT JOIN Auto_TestCaseTaskExecutions e ON r.execution_id = e.id - WHERE r.status = 'running' - AND r.start_time > DATE_SUB(NOW(), INTERVAL ? HOUR) - ORDER BY r.start_time ASC - LIMIT ?`, - [maxAgeHours, ExecutionRepository.ACTIVE_SLOTS_MAX_LIMIT] - ) as Array<{ id: number; taskId: number | null; startTime: Date | null }>; - return rows; - } - - /** - * 将指定 executionId 下所有 status=error 的预创建记录批量更新为目标状态 - * 同时填充 start_time(若为 NULL 则用 NOW())和 duration(若为 NULL 则用 0) - */ - async bulkUpdateErrorResults(executionId: number, targetStatus: 'passed' | 'failed' | 'skipped'): Promise { - // 只更新 status 和 end_time,不填充 start_time 和 duration: - // 这些占位符记录没有真实的执行时间和耗时数据,保留 NULL 让前端显示 "-", - // 避免用当前时间或 0 误导用户。 - const result = await this.testRunResultRepository.query(` - UPDATE Auto_TestRunResults - SET status = ?, - end_time = COALESCE(end_time, NOW()) - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - `, [targetStatus, executionId]) as { affectedRows?: number; changedRows?: number }; - return result?.affectedRows ?? 0; - } - - /** - * 统计指定 executionId 下各状态的结果数量 - */ - async countResultsByStatus(executionId: number): Promise<{ passed: number; failed: number; skipped: number; total: number }> { - const rows = await this.testRunResultRepository.query(` - SELECT - SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, - SUM(CASE WHEN status IN ('failed', 'error') THEN 1 ELSE 0 END) AS failed, - SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, - COUNT(*) AS total - FROM Auto_TestRunResults - WHERE execution_id = ? - `, [executionId]) as Array<{ passed: string; failed: string; skipped: string; total: string }>; - - if (!rows || rows.length === 0) { - return { passed: 0, failed: 0, skipped: 0, total: 0 }; - } - return { - passed: Number(rows[0].passed ?? 0), - failed: Number(rows[0].failed ?? 0), - skipped: Number(rows[0].skipped ?? 0), - total: Number(rows[0].total ?? 0), - }; - } - - /** - * 通过 executionId(Auto_TestCaseTaskExecutions.id)找到关联的 Auto_TestRun, - * 并同步更新其 status/passedCases/failedCases/skippedCases/durationMs 等统计字段。 - * 用于修复 handleCallback 路径(/api/executions/callback)只更新了 TaskExecution 而未更新 TestRun 的问题。 - */ - async syncTestRunByExecutionId(executionId: number, data: { - status: 'success' | 'failed' | 'cancelled' | 'aborted'; - passedCases: number; - failedCases: number; - skippedCases: number; - durationMs: number; - }): Promise { - // Auto_TestRun.execution_id 字段直接关联了 executionId - const testRun = await this.testRunRepository.findOne({ - where: { executionId }, - select: ['id', 'status'], - }); - - if (!testRun) { - logger.warn('syncTestRunByExecutionId: no TestRun found for executionId', { executionId }, LOG_CONTEXTS.REPOSITORY); - return false; - } - - const mappedStatus = this.mapStatusForTestRun(data.status); - await this.testRunRepository.update(testRun.id, { - status: mappedStatus, - passedCases: data.passedCases, - failedCases: data.failedCases, - skippedCases: data.skippedCases, - durationMs: data.durationMs, - endTime: new Date(), - }); - - logger.debug('syncTestRunByExecutionId: synced TestRun stats', { - executionId, - runId: testRun.id, - passedCases: data.passedCases, - failedCases: data.failedCases, - skippedCases: data.skippedCases, - }, LOG_CONTEXTS.REPOSITORY); - - return true; - } - - // ============================================================================ - // 私有辅助方法 - // ============================================================================ - - /** - * 辅助方法:将状态映射为 TestRun 的枚举值 - * @param status 输入状态 - * @returns TestRun 的状态枚举值 - */ - private mapStatusForTestRun(status: string): 'pending' | 'running' | 'success' | 'failed' | 'aborted' { - // 将 'cancelled' 映射为 'aborted' 以匹配 TestRun 的枚举 - if (status === 'cancelled') { - return 'aborted'; - } - return status as 'pending' | 'running' | 'success' | 'failed' | 'aborted'; - } - - /** - * 归一化回调中的单用例状态,避免非标准值导致写库失败并残留占位 error。 - */ - private normalizeCaseResultStatus(status: string): TestRunResultStatusType { - const normalized = String(status ?? '').trim().toLowerCase(); - - if (normalized === 'passed' || normalized === 'success' || normalized === 'pass') { - return TestRunResultStatus.PASSED; - } - - if (normalized === 'failed' || normalized === 'fail') { - return TestRunResultStatus.FAILED; - } - - if (normalized === 'skipped' || normalized === 'skip') { - return TestRunResultStatus.SKIPPED; - } - - if (normalized === 'error') { - return TestRunResultStatus.ERROR; - } - - // 未知状态统一归为 error,便于前端识别并排查 - return TestRunResultStatus.ERROR; - } - - /** - * 修复10: completeBatch 拆分 - 多策略解析 executionId - * - * 策略优先级(由高到低): - * 1. 从 Auto_TestRun.execution_id 直接查询(最可靠,run 与 execution 的强绑定) - * 2. 调用方传入的缓存值(仅在 run 尚未绑定 execution_id 时使用) - * 3. 时间窗口 + 触发者反查(兜底,可能不够精确) - */ - private async resolveExecutionIdForBatch(runId: number, cachedExecutionId?: number): Promise { - let runBoundExecutionId: number | undefined; - - // 优先读取 Auto_TestRun.execution_id,避免时间窗口误关联到其他执行 - try { - const trRows = await this.testRunRepository.query( - `SELECT execution_id FROM Auto_TestRun WHERE id = ? LIMIT 1`, - [runId] - ) as Array<{ execution_id: number | null }>; - if (trRows.length > 0 && trRows[0].execution_id) { - runBoundExecutionId = trRows[0].execution_id; - } - } catch (error) { - // 兼容旧库可能不存在 execution_id 列的场景 - logger.debug('Failed to read execution_id column, falling back to cache/time-window search', { - runId, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.REPOSITORY); - } - - if (runBoundExecutionId) { - if (cachedExecutionId && cachedExecutionId !== runBoundExecutionId) { - logger.warn('Cached executionId mismatch, using run-bound execution_id', { - runId, - cachedExecutionId, - runBoundExecutionId, - }, LOG_CONTEXTS.REPOSITORY); - } - logger.debug('Resolved executionId from Auto_TestRun.execution_id', { - runId, - resolvedId: runBoundExecutionId, - }, LOG_CONTEXTS.REPOSITORY); - return runBoundExecutionId; - } - - if (cachedExecutionId) { - logger.debug('Using cached executionId because run has no bound execution_id', { - runId, - cachedExecutionId, - }, LOG_CONTEXTS.REPOSITORY); - return cachedExecutionId; - } - - // 降级到时间窗口反查 - const fallbackId = await this.findExecutionIdByRunId(runId); - return fallbackId || undefined; - } - - /** - * 兜底策略:通过 Auto_TestRunResults 表中已有的占位记录反查 executionId - * - * 背景:triggerExecution 事务中会将 executionId 回填到 Auto_TestRun.execution_id, - * 同时也将 executionId 写入 Auto_TestRunResults 的每条预创建记录。 - * 若 Auto_TestRun.execution_id 字段 null(旧数据),时间窗口反查也失败时, - * 可以从 Auto_TestRunResults 里找到该 runId 对应的 executionId。 - * - * 实现:通过 Auto_TestRun.id 的创建时间找到在同时间段内插入的 TestRunResults, - * 从中读取 execution_id 作为 fallback。 - */ - private async resolveExecutionIdFromRunResults(runId: number): Promise { - try { - // 先查该 runId 对应的 TestRun 创建时间 - const runRows = await this.testRunRepository.query( - `SELECT id, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1`, - [runId] - ) as Array<{ id: number; created_at: Date }>; - - if (!runRows || runRows.length === 0) return undefined; - - const createdAt = runRows[0].created_at; - - // 在 TestRun 创建时间 ±120 秒内,查找 Auto_TestRunResults 中存在的 execution_id - // triggerExecution 事务中同一批次的 TestRunResults 会在 TestRun 创建后立即插入 - const resultRows = await this.testRunResultRepository.query( - `SELECT DISTINCT execution_id - FROM Auto_TestRunResults - WHERE created_at BETWEEN DATE_SUB(?, INTERVAL 120 SECOND) - AND DATE_ADD(?, INTERVAL 120 SECOND) - AND execution_id IS NOT NULL - ORDER BY id ASC - LIMIT 1`, - [createdAt, createdAt] - ) as Array<{ execution_id: number }>; - - if (resultRows && resultRows.length > 0 && resultRows[0].execution_id) { - logger.debug('resolveExecutionIdFromRunResults: found executionId via TestRunResults time-window', { - runId, - executionId: resultRows[0].execution_id, - }, LOG_CONTEXTS.REPOSITORY); - return resultRows[0].execution_id; - } - - return undefined; - } catch (error) { - logger.debug('resolveExecutionIdFromRunResults: query failed', { - runId, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.REPOSITORY); - return undefined; - } - } - - /** - * 修复10: completeBatch 拆分 - 同步 TaskExecution(Auto_TestCaseTaskExecutions)状态 - * - * completeBatch 主要更新 Auto_TestRun,此方法负责将状态同步到 Auto_TestCaseTaskExecutions, - * 保证 getRecentExecutions 查询 TaskExecution 时也能读到最新状态。 - */ - private async syncTaskExecutionStatus(resolvedExecutionId: number, results: BatchResults): Promise { - // TaskExecution 不支持 'aborted',统一映射为 'cancelled' - const taskExecStatus: 'success' | 'failed' | 'cancelled' = - results.status === 'aborted' || results.status === 'cancelled' ? 'cancelled' - : results.status === 'success' ? 'success' - : 'failed'; - - await this.repository.update(resolvedExecutionId, { - status: taskExecStatus, - passedCases: results.passedCases, - failedCases: results.failedCases, - skippedCases: results.skippedCases, - duration: Math.round(results.durationMs / 1000), - endTime: new Date(), - }); - } - - /** - * 修复10: completeBatch 拆分 - 处理带详细用例结果的更新路径 - * - * 遍历 caseResults,逐条尝试更新已有记录;若记录不存在则新建。 - * 失败的条目会被收集并记录警告日志,但不会中断其余条目的处理。 - */ - private async updateDetailedCaseResults( - runId: number, - executionId: number, - caseResults: NonNullable - ): Promise { - const failedResults: Array<{ caseId?: number; error: string }> = []; - - for (const result of caseResults) { - try { - const resolveTime = (v: string | number | undefined): Date | undefined => { - if (!v) return undefined; - const d = new Date(v); - return isNaN(d.getTime()) ? undefined : d; - }; - const startTime = resolveTime(result.startTime) ?? new Date(); - const endTime = resolveTime(result.endTime) ?? new Date(); - - const normalizedStatus = this.normalizeCaseResultStatus(result.status); - - const updated = await this.updateTestResult(executionId, result.caseId, { - status: normalizedStatus, - duration: result.duration, - errorMessage: result.errorMessage, - errorStack: result.stackTrace, - screenshotPath: result.screenshotPath, - logPath: result.logPath, - assertionsTotal: result.assertionsTotal, - assertionsPassed: result.assertionsPassed, - responseData: result.responseData, - startTime, - endTime, - caseName: result.caseName, // 无 caseId 时按 caseName fallback 匹配 - }); - - // 【诊断日志】记录每条用例的匹配结果,便于追踪明细写入失败的原因 - logger.debug( - `updateDetailedCaseResults: case update result`, - { - executionId, - runId, - caseId: result.caseId, - caseName: result.caseName, - status: normalizedStatus, - matched: updated, - action: updated ? 'updated' : (result.caseId !== undefined ? 'will_create' : 'skipped_no_caseId'), - }, - LOG_CONTEXTS.REPOSITORY - ); - - if (!updated) { - // 若 caseId 缺失(caseName fallback 场景),跳过 createTestResult 避免 DB NOT NULL 错误 - if (result.caseId !== undefined) { - await this.createTestResult({ - executionId, - caseId: result.caseId, - caseName: result.caseName, - status: normalizedStatus, - duration: result.duration, - errorMessage: result.errorMessage, - errorStack: result.stackTrace, - screenshotPath: result.screenshotPath, - logPath: result.logPath, - assertionsTotal: result.assertionsTotal, - assertionsPassed: result.assertionsPassed, - responseData: result.responseData, - startTime, - endTime, - }); - } - } - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - failedResults.push({ caseId: result.caseId, error: errorMsg }); - logger.error( - `Failed to process result for case`, - { caseId: result.caseId, executionId, error: errorMsg }, - LOG_CONTEXTS.REPOSITORY - ); - } - } - - if (failedResults.length > 0) { - logger.warn( - `Some results failed to process in batch`, - { - runId, - failedCount: failedResults.length, - totalCount: caseResults.length, - failedCaseIds: failedResults.map(f => f.caseId), - }, - LOG_CONTEXTS.REPOSITORY - ); - } - } - - /** - * 修复12: 清理残留的 ERROR 占位符 - * - * 场景:Jenkins 回调只返回部分测试结果(如网络故障、超时、脚本异常) - * 例如:10个用例只返回3个结果,剩余7个 ERROR 占位符不会被 updateDetailedCaseResults 清理 - * - * 解决方案: - * 1. 统计已更新的结果数量 vs 回调中的统计数 (passedCases + failedCases + skippedCases) - * 2. 计算预期的总用例数(数据库记录总数) - * 3. 检查是否有残留的 ERROR 占位符 - * 4. 根据整体执行状态批量更新残留占位符 - * - * @param runId TestRun ID - * @param executionId TaskExecution ID - * @param detailedResultsCount 回调中详细结果的数量 - * @param results 回调中的完整结果数据 - */ - private async cleanupResidualErrorPlaceholders( - runId: number, - executionId: number, - detailedResultsCount: number, - results: BatchResults - ): Promise { - // 1. 获取该 executionId 下的所有结果状态分布 - const statusCounts = await this.countResultsByStatus(executionId); - const totalFromCallback = results.passedCases + results.failedCases + results.skippedCases; - - // 2. 检查是否有残留的 ERROR 占位符 - const errorCountQuery = await this.testRunResultRepository.query(` - SELECT COUNT(*) AS errorCount - FROM Auto_TestRunResults - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - `, [executionId]) as Array<{ errorCount: string }>; - const residualErrorCount = Number(errorCountQuery[0]?.errorCount ?? 0); - - // 如果没有残留 ERROR 占位符,无需处理 - if (residualErrorCount === 0) { - logger.debug( - `No residual ERROR placeholders found, skip cleanup`, - { runId, executionId, detailedResultsCount, totalFromCallback }, - LOG_CONTEXTS.REPOSITORY - ); - return; - } - - // 3. 判断是否有部分结果缺失的情况 - // 如果回调中的统计数与详细结果数不一致,说明确实存在部分结果缺失 - const hasPartialResults = detailedResultsCount < totalFromCallback || statusCounts.total > detailedResultsCount; - - logger.info( - `Detected residual ERROR placeholders, will clean up based on overall status`, - { - runId, - executionId, - detailedResultsCount, - totalFromCallback, - dbTotal: statusCounts.total, - residualErrorCount, - hasPartialResults, - overallStatus: results.status, - }, - LOG_CONTEXTS.REPOSITORY - ); - - // 4. 根据整体执行状态批量更新残留的 ERROR 占位符 - // 安全策略:当存在部分结果缺失时,不应用"假定性"状态,避免假阳性 - // - 如果整体状态是 success 且没有部分结果缺失,将残留 ERROR 更新为 passed - // - 如果整体状态是 success 但有部分结果缺失,将残留 ERROR 更新为 skipped(安全处理) - // - 如果整体状态是 failed,将残留 ERROR 更新为 failed(保守处理) - // - 如果整体状态是 aborted/cancelled,将残留 ERROR 更新为 skipped - let targetStatus: 'passed' | 'failed' | 'skipped'; - let reason: string; - - if (results.status === 'success') { - if (hasPartialResults) { - // 安全处理:部分结果缺失时,残留 ERROR 可能是未执行的用例,标记为 skipped 避免假阳性 - targetStatus = 'skipped'; - reason = 'overall execution succeeded but partial results missing - marking as skipped to avoid false positives'; - } else { - // 完整结果场景:残留 ERROR 可能是 Jenkins 未返回的结果,假设它们通过了 - targetStatus = 'passed'; - reason = 'overall execution succeeded with complete results'; - } - } else if (results.status === 'failed') { - // 失败场景:保守处理,将残留 ERROR 标记为 failed - targetStatus = 'failed'; - reason = 'overall execution failed'; - } else { - // 取消/中断场景:将残留 ERROR 标记为 skipped - targetStatus = 'skipped'; - reason = 'execution was cancelled or aborted'; - } - - const updatedCount = await this.bulkUpdateErrorResults(executionId, targetStatus); - - logger.info( - `Cleaned up residual ERROR placeholders`, - { - runId, - executionId, - residualErrorCount, - updatedCount, - targetStatus, - reason, - hasPartialResults, - overallStatus: results.status, - }, - LOG_CONTEXTS.REPOSITORY - ); - - // 5. 同步更新统计数(确保数据库统计与实际状态一致) - const newCounts = await this.countResultsByStatus(executionId); - await this.testRunRepository.update(runId, { - passedCases: newCounts.passed, - failedCases: newCounts.failed, - skippedCases: newCounts.skipped, - }); - } - - /** - * 修复10 + 修复11: completeBatch 拆分 - 处理只有汇总统计、没有详细用例结果的兜底路径 - * - * 策略一(totalSummary > 0):Jenkins 只传了统计数(passedCases/failedCases/skippedCases > 0), - * 按顺序将预创建的 error 记录批量更新为对应状态。 - * 修复11: 改为按状态分组使用 IN 批量 UPDATE,替代原来的逐条循环,减少 SQL 往返次数。 - * - * 策略二(totalSummary = 0):Jenkins 统计数全为0, - * 先查数据库现有结果,若已有真实结果则回填统计;若全是 error 记录则按整体状态批量更新。 - */ - private async updateSummaryOnlyResults( - runId: number, - executionId: number, - results: BatchResults - ): Promise { - const totalSummary = results.passedCases + results.failedCases + results.skippedCases; - - if (totalSummary > 0) { - logger.info( - `No detailed results provided, updating pre-created records using summary counts`, - { runId, executionId, passedCases: results.passedCases, failedCases: results.failedCases, skippedCases: results.skippedCases }, - LOG_CONTEXTS.REPOSITORY - ); - - // 【修复】只获取仍处于 error 状态的占位记录,避免覆盖已被 updateDetailedCaseResults 写入的真实结果 - // 原查询无 status 过滤,在并发/重试场景下会把已正确更新的记录重置为错误状态 - const preCreatedResults = await this.testRunResultRepository.query(` - SELECT id FROM Auto_TestRunResults - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - ORDER BY id ASC - `, [executionId]) as Array<{ id: number }>; - - const now = new Date(); - const batchUpdate = async (ids: number[], status: 'passed' | 'failed' | 'skipped') => { - if (ids.length === 0) return; - // 只更新 status 和 end_time,不填充 start_time 和 duration: - // 没有真实执行数据时,保留 NULL 让前端显示 "-" - const placeholders = ids.map(() => '?').join(', '); - await this.testRunResultRepository.query( - `UPDATE Auto_TestRunResults - SET status = ?, - end_time = COALESCE(end_time, ?) - WHERE id IN (${placeholders})`, - [status, now, ...ids] - ); - }; - - if (preCreatedResults.length > 0) { - const passedEnd = results.passedCases; - const failedEnd = passedEnd + results.failedCases; - - // 修复11: 按状态分组收集 id,再批量 UPDATE,替代原来的逐条循环 - const passedIds = preCreatedResults.slice(0, passedEnd).map(r => r.id); - const failedIds = preCreatedResults.slice(passedEnd, failedEnd).map(r => r.id); - const skippedIds = preCreatedResults.slice(failedEnd).map(r => r.id); - - logger.info( - `updateSummaryOnlyResults: distributing status to error placeholders`, - { - runId, - executionId, - totalErrorPlaceholders: preCreatedResults.length, - passedIds: passedIds.length, - failedIds: failedIds.length, - skippedIds: skippedIds.length, - summaryPassed: results.passedCases, - summaryFailed: results.failedCases, - summarySkipped: results.skippedCases, - }, - LOG_CONTEXTS.REPOSITORY - ); - - await Promise.all([ - batchUpdate(passedIds, 'passed'), - batchUpdate(failedIds, 'failed'), - batchUpdate(skippedIds, 'skipped'), - ]); - - logger.info( - `Updated pre-created result records using summary counts`, - { runId, executionId, updatedCount: preCreatedResults.length }, - LOG_CONTEXTS.REPOSITORY - ); - } else { - // 【修复】当 error 占位符已被轮询路径提前清理(改为 failed/passed)时, - // 若此次回调汇总数据与数据库明细记录不一致,需要重新修正以保持数据同步。 - // 场景:轮询路径在无 JUnit 结果时将占位符改为 failed,随后回调到达告知 passedCases>0。 - // 此时需要将 failed 记录中多余的(按顺序)重新分配为 passed。 - const currentCounts = await this.testRunResultRepository.query(` - SELECT - SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, - SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed, - SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, - COUNT(*) AS total - FROM Auto_TestRunResults - WHERE execution_id = ? - `, [executionId]) as Array<{ passed: string; failed: string; skipped: string; total: string }>; - - if (currentCounts.length > 0) { - const dbPassed = Number(currentCounts[0].passed ?? 0); - const dbFailed = Number(currentCounts[0].failed ?? 0); - const dbTotal = Number(currentCounts[0].total ?? 0); - - const expectedPassed = results.passedCases; - const expectedFailed = results.failedCases; - - // 只在汇总数据与数据库明细不一致时进行修正(避免覆盖正确数据) - const needsReconcile = dbTotal > 0 && (dbPassed !== expectedPassed || dbFailed !== expectedFailed); - - if (needsReconcile) { - logger.info( - `updateSummaryOnlyResults: no error placeholders but counts mismatch, reconciling`, - { - runId, - executionId, - dbPassed, - dbFailed, - dbTotal, - expectedPassed, - expectedFailed, - summaryPassed: results.passedCases, - summaryFailed: results.failedCases, - }, - LOG_CONTEXTS.REPOSITORY - ); - - // 若回调说全部通过(passedCases == total),但 DB 里全是 failed, - // 则将所有非 passed 记录按序重新分配为 passed - if (expectedPassed > 0 && dbPassed < expectedPassed) { - // 查出多余的 failed 记录,按顺序将前 (expectedPassed - dbPassed) 条改为 passed - const needMorePassed = expectedPassed - dbPassed; - const failedRecords = await this.testRunResultRepository.query(` - SELECT id FROM Auto_TestRunResults - WHERE execution_id = ? AND status = 'failed' - ORDER BY id ASC - LIMIT ? - `, [executionId, needMorePassed]) as Array<{ id: number }>; - - if (failedRecords.length > 0) { - const ids = failedRecords.map(r => r.id); - await batchUpdate(ids, 'passed'); - logger.info( - `updateSummaryOnlyResults: reconciled failed→passed records`, - { runId, executionId, reconciled: ids.length, expectedPassed, dbPassed }, - LOG_CONTEXTS.REPOSITORY - ); - } - } else if (expectedFailed > 0 && dbFailed < expectedFailed) { - // 反向场景:DB 里 passed 多,但 Jenkins 说有 failed - const needMoreFailed = expectedFailed - dbFailed; - const passedRecords = await this.testRunResultRepository.query(` - SELECT id FROM Auto_TestRunResults - WHERE execution_id = ? AND status = 'passed' - ORDER BY id DESC - LIMIT ? - `, [executionId, needMoreFailed]) as Array<{ id: number }>; - - if (passedRecords.length > 0) { - const ids = passedRecords.map(r => r.id); - await batchUpdate(ids, 'failed'); - logger.info( - `updateSummaryOnlyResults: reconciled passed→failed records`, - { runId, executionId, reconciled: ids.length, expectedFailed, dbFailed }, - LOG_CONTEXTS.REPOSITORY - ); - } - } - } - } - } - } else { - // 统计数全为0,且没有详细 results 数组 - // 先查数据库里该 executionId 下的现有记录状态 - const countRows = await this.testRunResultRepository.query(` - SELECT - SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, - SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed, - SUM(CASE WHEN status IS NULL OR status = 'error' THEN 1 ELSE 0 END) AS error_count, - SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, - COUNT(*) AS total - FROM Auto_TestRunResults - WHERE execution_id = ? - `, [executionId]) as Array<{ passed: string; failed: string; error_count: string; skipped: string; total: string }>; - - if (countRows.length > 0) { - const dbPassed = Number(countRows[0].passed ?? 0); - const dbFailed = Number(countRows[0].failed ?? 0); - const dbError = Number(countRows[0].error_count ?? 0); - const dbSkipped = Number(countRows[0].skipped ?? 0); - const dbTotal = Number(countRows[0].total ?? 0); - const dbFinished = dbPassed + dbFailed + dbSkipped; - - if (dbTotal > 0 && dbFinished > 0) { - // 已有非 error 的真实结果,直接用统计值回填 Auto_TestRun,不覆盖为0 - await this.testRunRepository.update(runId, { - passedCases: dbPassed, - failedCases: dbFailed, - skippedCases: dbSkipped, - }); - logger.info( - `Recalculated stats from existing results and back-filled Auto_TestRun`, - { runId, executionId, dbPassed, dbFailed, dbSkipped }, - LOG_CONTEXTS.REPOSITORY - ); - } else if (dbTotal > 0 && dbError > 0) { - // 全是预创建的 error 记录,根据执行整体状态将它们批量更新 - // success → passed;failed/aborted/cancelled → failed - const mappedStatus: 'passed' | 'failed' = - (results.status === 'success') ? 'passed' : 'failed'; - - // 只更新 status 和 end_time,不填充 start_time 和 duration,保留 NULL 让前端正确显示 "-" - await this.testRunResultRepository.query(` - UPDATE Auto_TestRunResults - SET status = ?, - end_time = COALESCE(end_time, NOW()) - WHERE execution_id = ? AND (status IS NULL OR status = 'error') - `, [mappedStatus, executionId]); - - // 根据映射结果更新 Auto_TestRun 统计 - const newPassed = mappedStatus === 'passed' ? dbError : 0; - const newFailed = mappedStatus === 'failed' ? dbError : 0; - await this.testRunRepository.update(runId, { - passedCases: newPassed, - failedCases: newFailed, - skippedCases: 0, - }); - - logger.info( - `Bulk-updated pre-created error records based on overall execution status`, - { runId, executionId, mappedStatus, updatedCount: dbError, newPassed, newFailed }, - LOG_CONTEXTS.REPOSITORY - ); - } - } - } - } -} +import { ExecutionRepositoryMaintenance } from './ExecutionRepositoryMaintenance'; + +export type { + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TaskExecutionWithUser, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; + +export class ExecutionRepository extends ExecutionRepositoryMaintenance {} diff --git a/server/repositories/ExecutionRepositoryBase.ts b/server/repositories/ExecutionRepositoryBase.ts new file mode 100644 index 0000000..bdb0f93 --- /dev/null +++ b/server/repositories/ExecutionRepositoryBase.ts @@ -0,0 +1,643 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; +import type { + BatchResults, + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TaskExecutionWithUser, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; + +export abstract class ExecutionRepositoryBase extends BaseRepository { + + // 修复2: 为私有属性添加 readonly 修饰符,防止意外重赋值 + protected readonly testRunRepository: Repository; + protected readonly testRunResultRepository: Repository; + protected readonly testCaseRepository: Repository; + protected readonly userRepository: Repository; + + // 修复8: 提取魔法数字为类静态常量,方便统一维护 + /** 批量插入的批次大小:经过性能测试,100 条/批在 MySQL 上表现最佳 */ + protected static readonly BATCH_INSERT_SIZE = 100; + /** 时间窗口反查的容差(秒):允许 TestRun 与 TaskExecution 创建时间差在 ±120s 内 */ + protected static readonly TIME_WINDOW_TOLERANCE_SECONDS = 120; + /** 扩大时间窗口兜底容差(秒):±300s 的宽松窗口,作为最后兜底 */ + protected static readonly TIME_WINDOW_FALLBACK_SECONDS = 300; + /** getActiveRunningSlots 返回的最大记录数,避免单次查询过多 */ + protected static readonly ACTIVE_SLOTS_MAX_LIMIT = 200; + + abstract findExecutionIdByRunId(runId: number): Promise; + + constructor(dataSource: DataSource) { + super(dataSource, TaskExecution); + this.testRunRepository = dataSource.getRepository(TestRun); + this.testRunResultRepository = dataSource.getRepository(TestRunResult); + this.testCaseRepository = dataSource.getRepository(TestCase); + this.userRepository = dataSource.getRepository(User); + } + + /** + * 创建测试运行记录 + */ + async createTestRun(runData: { + projectId: number; + triggerType: TestRunTriggerTypeType; + /** null 表示系统调度触发(无操作人) */ + triggerBy: number | null; + jenkinsJob?: string; + runConfig?: Record; + totalCases: number; + }): Promise { + const testRun = this.testRunRepository.create({ + ...runData, + status: TestRunStatus.PENDING, + }); + return this.testRunRepository.save(testRun); + } + + /** + * 创建任务运行记录 + */ + async createTaskExecution(executionData: { + taskId?: number; + taskName?: string; + totalCases: number; + /** null 表示系统调度触发(无操作人) */ + executedBy: number | null; + }): Promise { + const execution = this.repository.create({ + ...executionData, + status: TaskExecutionStatus.PENDING, + }); + return this.repository.save(execution); + } + + /** + * 批量创建测试结果记录 + * 性能优化: + * - 使用 insert 而非逐个 save,减少数据库往返 + * - 修复6: 按批次创建实体对象,避免一次性占用大量内存 + */ + async createTestResults( + results: Array<{ + executionId: number; + caseId: number; + caseName: string; + status: TestRunResultStatusType | null; + duration?: number; + errorMessage?: string; + errorStack?: string; + screenshotPath?: string; + logPath?: string; + assertionsTotal?: number; + assertionsPassed?: number; + responseData?: string; + }> + ): Promise { + if (results.length === 0) { + return; + } + + // 修复6: 按批次创建实体并插入,避免一次性将全部数据加载到内存 + for (let i = 0; i < results.length; i += ExecutionRepositoryBase.BATCH_INSERT_SIZE) { + const batch = results.slice(i, i + ExecutionRepositoryBase.BATCH_INSERT_SIZE); + const entities = batch.map(result => + this.testRunResultRepository.create(result) + ); + await this.testRunResultRepository.insert(entities); + } + } + + /** + * 更新执行状态为运行中 + * 同时清除 endTime,防止从终态回退时出现数据不一致 + */ + async markExecutionRunning(executionId: number): Promise { + await this.repository.update(executionId, { + status: TaskExecutionStatus.RUNNING, + startTime: new Date(), + endTime: null as unknown as Date, // 清除可能存在的旧 endTime,确保数据一致性 + }); + } + + /** + * 获取执行详情 + */ + async getExecutionDetail(executionId: number): Promise { + const execution = await this.repository.createQueryBuilder('execution') + .leftJoinAndSelect('execution.executedByUser', 'user') + .where('execution.id = :executionId', { executionId }) + .getOne(); + + if (!execution) { + return null; + } + + const results = await this.testRunResultRepository.find({ + where: { executionId }, + order: { id: 'ASC' }, + }); + + // 安全地获取用户名 + const executionWithUser = execution as TaskExecutionWithUser; + const executedByName = executionWithUser.executedByUser?.displayName + || executionWithUser.executedByUser?.username + || undefined; + + return { + execution: { + ...execution, + executedByName, + }, + results, + }; + } + + /** + * 执行事务 - 公开方法供Service层使用 + */ + async runInTransaction(callback: (queryRunner: QueryRunner) => Promise): Promise { + return this.executeInTransaction(callback); + } + + /** + * 获取最近运行记录 + * 修复5: 修正 getRawMany() 返回原始字段名(execution_xxx 格式),显式映射为 RecentExecution 接口字段 + */ + async getRecentExecutions(limit: number = 10): Promise { + const rawRows = await this.repository.createQueryBuilder('execution') + .leftJoin('execution.executedByUser', 'user') + .select([ + 'execution.id', + 'execution.taskId', + 'execution.taskName', + 'execution.status', + 'execution.totalCases', + 'execution.passedCases', + 'execution.failedCases', + 'execution.skippedCases', + 'execution.duration', + 'execution.executedBy', + 'execution.startTime', + 'execution.endTime', + 'user.displayName', + 'user.username', + ]) + .orderBy('execution.startTime', 'DESC') + .limit(limit) + .getRawMany(); + + // getRawMany() 返回的字段名带有 alias 前缀(如 execution_id),需要显式映射 + return rawRows.map(raw => ({ + id: raw.execution_id, + taskId: raw.execution_taskId ?? raw.execution_task_id, + taskName: raw.execution_taskName ?? raw.execution_task_name, + status: raw.execution_status, + totalCases: raw.execution_totalCases ?? raw.execution_total_cases, + passedCases: raw.execution_passedCases ?? raw.execution_passed_cases, + failedCases: raw.execution_failedCases ?? raw.execution_failed_cases, + skippedCases: raw.execution_skippedCases ?? raw.execution_skipped_cases, + duration: raw.execution_duration, + executedBy: raw.execution_executedBy ?? raw.execution_executed_by, + executedByName: raw.user_displayName ?? raw.user_display_name ?? raw.user_username ?? undefined, + startTime: raw.execution_startTime ?? raw.execution_start_time, + endTime: raw.execution_endTime ?? raw.execution_end_time, + })); + } + + /** + * 取消执行 + * 修复3: 使用枚举常量替代硬编码字符串,保持与类型系统的一致性 + */ + async cancelExecution(executionId: number): Promise { + await this.repository.update( + { id: executionId, status: In([TaskExecutionStatus.PENDING, TaskExecutionStatus.RUNNING]) }, + { + status: TaskExecutionStatus.CANCELLED, + endTime: new Date(), + } + ); + } + + /** + * 更新执行结果统计 + */ + async updateExecutionResults( + executionId: number, + results: { + status: 'success' | 'failed' | 'cancelled'; + passedCases: number; + failedCases: number; + skippedCases: number; + duration: number; + } + ): Promise { + await this.repository.update(executionId, { + ...results, + endTime: new Date(), + }); + } + + /** + * 更新测试运行结果 + */ + async updateTestRunResults( + runId: number, + results: { + status: 'success' | 'failed' | 'cancelled'; + passedCases: number; + failedCases: number; + skippedCases: number; + durationMs: number; + } + ): Promise { + await this.testRunRepository.update(runId, { + status: results.status === 'cancelled' ? 'aborted' : results.status, + passedCases: results.passedCases, + failedCases: results.failedCases, + skippedCases: results.skippedCases, + durationMs: results.durationMs, + endTime: new Date(), + }); + } + + /** + * 获取活跃用例信息 + */ + async getActiveCases(caseIds: number[]): Promise { + return this.testCaseRepository.find({ + where: { + id: In(caseIds), + enabled: true, + }, + select: ['id', 'name', 'type', 'scriptPath'], + }); + } + + /** + * 更新 Jenkins 构建信息 + */ + async updateJenkinsInfo( + runId: number, + jenkinsInfo: { + buildId: string; + buildUrl: string; + } + ): Promise { + const jobMatch = jenkinsInfo.buildUrl.match(/\/job\/([^/]+)\//); + const updateData: QueryDeepPartialEntity = { + jenkinsBuildId: jenkinsInfo.buildId, + jenkinsUrl: jenkinsInfo.buildUrl, + status: TestRunStatus.RUNNING, + startTime: new Date(), + }; + + if (jobMatch) { + updateData.jenkinsJob = jobMatch[1]; + } + + await this.testRunRepository.update(runId, updateData); + } + + /** + * 获取测试运行详情 + */ + async getTestRunDetail(runId: number): Promise { + const testRun = await this.testRunRepository.createQueryBuilder('testRun') + .leftJoinAndSelect('testRun.triggerByUser', 'user') + .where('testRun.id = :runId', { runId }) + .getOne(); + + if (!testRun) { + return null; + } + + // 安全地获取用户名 + const testRunWithUser = testRun as TestRunWithUser; + const triggerByName = testRunWithUser.triggerByUser?.displayName + || testRunWithUser.triggerByUser?.username + || undefined; + + return { + ...testRun, + triggerByName, + }; + } + + /** + * 获取测试运行详情(返回 snake_case 格式,与 TestRunRecord 接口兼容) + */ + async getTestRunDetailRow(runId: number): Promise { + const rows: TestRunRow[] = await this.testRunRepository.query(` + SELECT tr.id, tr.project_id, + CASE WHEN tr.project_id IS NOT NULL THEN CONCAT("项目 #", tr.project_id) ELSE "未分类" END as project_name, + tr.status, tr.trigger_type, tr.trigger_by, + COALESCE(u.display_name, u.username, "系统") as trigger_by_name, + tr.jenkins_job, tr.jenkins_build_id, tr.jenkins_url, + JSON_UNQUOTE(JSON_EXTRACT(tr.run_config, '$.abortReason')) AS abort_reason, + tr.total_cases, tr.passed_cases, tr.failed_cases, tr.skipped_cases, + tr.duration_ms, tr.start_time, tr.end_time, tr.created_at + FROM Auto_TestRun tr + LEFT JOIN Auto_Users u ON tr.trigger_by = u.id + WHERE tr.id = ? + `, [runId]); + return rows[0] ?? null; + } + + + /** + * 获取执行结果列表(支持分页与服务端筛选) + * @param executionId 执行ID + * @param options 分页与筛选参数 + */ + async getExecutionResults( + executionId: number, + options: { + page?: number; + pageSize?: number; + status?: string; + keyword?: string; + } = {} + ): Promise<{ data: ExecutionResultRow[]; total: number }> { + const page = Math.max(1, options.page ?? 1); + const pageSize = Math.min(100, Math.max(1, options.pageSize ?? 20)); + const offset = (page - 1) * pageSize; + + const conditions: string[] = ["r.execution_id = ?"]; + const params: (string | number)[] = [executionId]; + + if (options.status && options.status !== "all") { + // "failed" 筛选同时包含 error 状态(Jenkins 执行异常写入 error,前端统一视为失败) + if (options.status === "failed") { + conditions.push("r.status IN ('failed', 'error')"); + } else if (options.status === "pending") { + conditions.push("r.status IS NULL"); + } else { + conditions.push("r.status = ?"); + params.push(options.status); + } + } + + if (options.keyword && options.keyword.trim()) { + conditions.push("(r.case_name LIKE ? OR COALESCE(tc.module, '') LIKE ?)"); + const like = `%${options.keyword.trim()}%`; + params.push(like, like); + } + + const whereClause = `WHERE ${conditions.join(" AND ")}`; + + const data: ExecutionResultRow[] = await this.testRunResultRepository.query(` + SELECT + r.id, + r.execution_id, + r.case_id, + r.case_name, + COALESCE(tc.module, "-") as module, + COALESCE(tc.priority, "P2") as priority, + COALESCE(tc.type, "api") as type, + COALESCE(r.status, 'pending') AS status, + r.start_time, + r.end_time, + r.duration, + r.error_message, + r.error_stack, + r.screenshot_path, + r.log_path, + r.assertions_total, + r.assertions_passed, + r.response_data, + r.created_at + FROM Auto_TestRunResults r + LEFT JOIN Auto_TestCase tc ON r.case_id = tc.id + ${whereClause} + ORDER BY r.id ASC + LIMIT ? OFFSET ? + `, [...params, pageSize, offset]); + + const countResult = await this.testRunResultRepository.query(` + SELECT COUNT(*) as total + FROM Auto_TestRunResults r + LEFT JOIN Auto_TestCase tc ON r.case_id = tc.id + ${whereClause} + `, params); + const total = Number(countResult[0]?.total ?? 0); + + return { data, total }; + } + + /** + * 根据 runId 查询该批次的用例执行结果(支持分页与筛选) + * 查询策略: + * 1. 优先读 Auto_TestRun.execution_id(新数据,直接关联) + * 2. 降级到时间窗口(±120秒)+ 触发者反查 TaskExecution.id + * 3. 兜底:在 ±300秒 窗口内,取同一触发者最近的 TaskExecution,不经过中间表 + */ + async getResultsByRunId( + runId: number, + options: { page?: number; pageSize?: number; status?: string; keyword?: string; } = {} + ): Promise<{ data: ExecutionResultRow[]; total: number }> { + // 策略1:从 Auto_TestRun.execution_id 直接读(需要该字段存在且非 NULL) + let executionId: number | null = null; + try { + const testRun = await this.testRunRepository.findOne({ where: { id: runId }, select: ["executionId"] }); + executionId = testRun?.executionId ?? null; + } catch (error) { + // 修复9: 添加 debug 日志,方便排查字段不存在等问题 + logger.debug('Failed to query execution_id column from Auto_TestRun, falling back to time-window search', { + runId, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.REPOSITORY); + } + + // 策略2:降级到时间窗口(±120秒)+ 触发者反查 + if (!executionId) { + executionId = await this.findExecutionIdByRunId(runId); + } + + // 找到 executionId,走正常路径 + if (executionId) { + return this.getExecutionResults(executionId, options); + } + + // 策略3:扩大时间窗口兜底(±300秒),不要求精确匹配,取最近的 TaskExecution + logger.warn("Fallback: trying extended time-window search for runId results", { runId }, LOG_CONTEXTS.REPOSITORY); + + const runRows = await this.testRunRepository.query(` + SELECT id, trigger_by, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1 + `, [runId]) as Array<{ id: number; trigger_by: number; created_at: Date }>; + + if (!runRows || runRows.length === 0) { + logger.warn("Cannot find runId, returning empty results", { runId }, LOG_CONTEXTS.REPOSITORY); + return { data: [], total: 0 }; + } + + const runCreatedAt = runRows[0].created_at; + const triggerBy = runRows[0].trigger_by; + + // 在 ±TIME_WINDOW_FALLBACK_SECONDS 窗口内,找同一触发者最近的 TaskExecution + const fallbackRows = await this.testRunResultRepository.query(` + SELECT e.id as execution_id + FROM Auto_TestCaseTaskExecutions e + WHERE e.executed_by = ? + AND e.created_at BETWEEN DATE_SUB(?, INTERVAL ? SECOND) AND DATE_ADD(?, INTERVAL ? SECOND) + ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC + LIMIT 1 + `, [ + triggerBy, + runCreatedAt, ExecutionRepositoryBase.TIME_WINDOW_FALLBACK_SECONDS, + runCreatedAt, ExecutionRepositoryBase.TIME_WINDOW_FALLBACK_SECONDS, + runCreatedAt, + ]) as Array<{ execution_id: number }>; + + if (fallbackRows && fallbackRows.length > 0 && fallbackRows[0].execution_id) { + const fallbackExecutionId = fallbackRows[0].execution_id; + logger.info("Extended fallback found executionId", { runId, fallbackExecutionId }, LOG_CONTEXTS.REPOSITORY); + return this.getExecutionResults(fallbackExecutionId, options); + } + + logger.warn("All strategies failed to find executionId for runId, returning empty results", { runId }, LOG_CONTEXTS.REPOSITORY); + return { data: [], total: 0 }; + } + + /** + * 获取所有测试运行记录(分页 + 筛选) + * 支持按触发方式、状态、时间范围筛选 + */ + async getAllTestRuns( + limit: number = 50, + offset: number = 0, + filters: { + triggerType?: string[]; + status?: string[]; + startDate?: string; + endDate?: string; + } = {} + ): Promise<{ data: TestRunRow[]; total: number }> { + // 动态拼接 WHERE 条件 + const conditions: string[] = []; + const params: (string | number)[] = []; + + if (filters.triggerType?.length) { + const placeholders = filters.triggerType.map(() => '?').join(', '); + conditions.push('tr.trigger_type IN (' + placeholders + ')'); + params.push(...filters.triggerType); + } + + if (filters.status?.length) { + const placeholders = filters.status.map(() => '?').join(', '); + conditions.push('tr.status IN (' + placeholders + ')'); + params.push(...filters.status); + } + + if (filters.startDate) { + // 报表页展示的是触发时间,因此筛选也按 created_at 对齐 + conditions.push('tr.created_at >= ?'); + params.push(`${filters.startDate} 00:00:00`); + } + + if (filters.endDate) { + // 结束日期:当天 23:59:59(北京时间) + conditions.push('tr.created_at <= ?'); + params.push(`${filters.endDate} 23:59:59`); + } + + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + // 数据查询 + const data = await this.testRunRepository.query(` + SELECT + tr.id, + tr.project_id, + CASE + WHEN tr.project_id IS NOT NULL THEN CONCAT('项目 #', tr.project_id) + ELSE '未分类' + END as project_name, + tr.status, + tr.trigger_type, + tr.trigger_by, + COALESCE(u.display_name, u.username, '系统') as trigger_by_name, + tr.jenkins_job, + tr.jenkins_build_id, + tr.jenkins_url, + JSON_UNQUOTE(JSON_EXTRACT(tr.run_config, '$.abortReason')) AS abort_reason, + tr.total_cases, + tr.passed_cases, + tr.failed_cases, + tr.skipped_cases, + tr.duration_ms, + tr.created_at, + tr.start_time, + tr.end_time + FROM Auto_TestRun tr + LEFT JOIN Auto_Users u ON tr.trigger_by = u.id + ${whereClause} + ORDER BY tr.id DESC + LIMIT ? OFFSET ? + `, [...params, limit, offset]); + + // 总数查询(同样带上筛选条件) + const countResult = await this.testRunRepository.query(` + SELECT COUNT(*) as total + FROM Auto_TestRun tr + ${whereClause} + `, params); + const total = countResult[0]?.total || 0; + + return { data, total }; + } + + /** + * 获取运行的用例列表 + * 性能优化:使用 JOIN 查询而非分步查询,消除 N+1 问题 + */ + async getRunCases(runId: number): Promise { + // 优化:使用一次 JOIN 查询获取所有数据,避免 N+1 问题 + const cases = await this.testCaseRepository + .createQueryBuilder('testCase') + .innerJoin( + 'Auto_TestRunResults', + 'result', + 'result.case_id = testCase.id AND result.execution_id = :runId', + { runId } + ) + .where('testCase.enabled = :enabled', { enabled: true }) + .select([ + 'testCase.id', + 'testCase.name', + 'testCase.type', + 'testCase.module', + 'testCase.priority', + 'testCase.scriptPath', + 'testCase.config', + ]) + .distinct(true) + .getMany(); + + return cases; + } + + /** + * 查找执行ID + * @deprecated 使用 {@link findExecutionIdByRunId} 替代,此方法只返回最新的 executionId,不够精确 + * @see findExecutionIdByRunId + */ +} diff --git a/server/repositories/ExecutionRepositoryBatch.ts b/server/repositories/ExecutionRepositoryBatch.ts new file mode 100644 index 0000000..2a9fa6a --- /dev/null +++ b/server/repositories/ExecutionRepositoryBatch.ts @@ -0,0 +1,976 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; +import { ExecutionRepositoryBase } from './ExecutionRepositoryBase'; +import { ExecutionRepositoryStatusUtilities } from './ExecutionRepositoryStatusUtilities'; +import type { + BatchResults, + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; +export abstract class ExecutionRepositoryBatch extends ExecutionRepositoryStatusUtilities { + abstract bulkUpdateErrorResults(executionId: number, targetStatus: 'passed' | 'failed' | 'skipped'): Promise; + + async completeBatch( + runId: number, + results: BatchResults, + executionId?: number + ): Promise { + // 0. 二次校验:防止并发回调导致正确结果被错误数据覆盖 + // 注意:这里不能使用 FOR UPDATE + 事务外 Repository 混用, + // 否则会出现“当前事务持锁、另一个连接更新同一行”导致 lock wait timeout。 + // 改为轻量读取 + 幂等防回退,避免锁竞争。 + const lockResult = await this.testRunRepository.query(` + SELECT id, status, passed_cases, failed_cases, skipped_cases + FROM Auto_TestRun + WHERE id = ? + LIMIT 1 + `, [runId]) as Array<{ + id: number; + status: string; + passed_cases: number | null; + failed_cases: number | null; + skipped_cases: number | null; + }>; + + const currentTestRun = lockResult.length > 0 ? { + id: lockResult[0].id, + status: lockResult[0].status, + passedCases: lockResult[0].passed_cases, + failedCases: lockResult[0].failed_cases, + skippedCases: lockResult[0].skipped_cases, + } : null; + + const finalStatuses = ['success', 'failed', 'cancelled', 'aborted']; + const hasDetailedResults = Array.isArray(results.results) && results.results.length > 0; + const hasSummaryCounts = (results.passedCases + results.failedCases + results.skippedCases) > 0; + + if (currentTestRun && finalStatuses.includes(currentTestRun.status)) { + // 数据版本检查:判断新数据是否比现有数据"更好" + const currentHasRealData = (currentTestRun.passedCases ?? 0) > 0 || + (currentTestRun.failedCases ?? 0) > 0 || + (currentTestRun.skippedCases ?? 0) > 0; + + // 终态已存在真实数据时,检查新数据是否会导致数据"倒退" + // 修复:无论新回调是否有详细结果,都要检查数据质量 + if (currentHasRealData) { + const currentTotal = (currentTestRun.passedCases ?? 0) + + (currentTestRun.failedCases ?? 0) + + (currentTestRun.skippedCases ?? 0); + const newTotal = results.passedCases + results.failedCases + results.skippedCases; + const currentPassed = currentTestRun.passedCases ?? 0; + const newPassed = results.passedCases; + + // 判断数据是否变差: + // 1. 总量减少(测试结果不完整) + // 2. 总量相同但 passed 减少(正确结果被错误结果覆盖) + // 3. 无详细结果且回调数据总量相同或更少(可能是空回调或部分数据) + const isDataRegression = newTotal < currentTotal || + (newTotal === currentTotal && newPassed < currentPassed) || + (!hasDetailedResults && newTotal <= currentTotal); + + if (isDataRegression) { + logger.warn( + 'completeBatch: rejected regression update - existing data is better than new callback', + { + runId, + currentStatus: currentTestRun.status, + currentPassed: currentTestRun.passedCases, + currentFailed: currentTestRun.failedCases, + currentSkipped: currentTestRun.skippedCases, + newPassed: results.passedCases, + newFailed: results.failedCases, + newSkipped: results.skippedCases, + hasDetailedResults, + newTotal, + currentTotal, + source: 'concurrent_callback_protection' + }, + LOG_CONTEXTS.REPOSITORY + ); + return; // 拒绝更新,保留现有的正确数据 + } + } + + logger.info( + 'completeBatch: allowing update to completed run with new payload', + { + runId, + currentStatus: currentTestRun.status, + hasDetailedResults, + hasSummaryCounts, + source: 'concurrent_callback_protection' + }, + LOG_CONTEXTS.REPOSITORY + ); + } + + // 1. 更新 TestRun 记录(将 cancelled 映射为 aborted 以兼容数据库枚举) + const mappedStatus = this.mapStatusForTestRun(results.status); + await this.testRunRepository.update(runId, { + status: mappedStatus, + passedCases: results.passedCases, + failedCases: results.failedCases, + skippedCases: results.skippedCases, + durationMs: results.durationMs, + endTime: new Date(), + }); + + // 2. 多策略解析 executionId + const resolvedExecutionId = await this.resolveExecutionIdForBatch(runId, executionId); + + // 【诊断日志】记录 executionId 解析结果,便于追踪「汇总 passed 但明细 FAILED」问题 + logger.info( + 'completeBatch: executionId resolution result', + { + runId, + cachedExecutionId: executionId, + resolvedExecutionId, + hasResults: !!(results.results && results.results.length > 0), + resultsCount: results.results?.length ?? 0, + summaryPassed: results.passedCases, + summaryFailed: results.failedCases, + summarySkipped: results.skippedCases, + }, + LOG_CONTEXTS.REPOSITORY + ); + + // 3. 同步 TaskExecution 状态(与 TestRun 保持一致) + if (resolvedExecutionId) { + await this.syncTaskExecutionStatus(resolvedExecutionId, results); + } + + // 4. 更新详细用例结果 + if (resolvedExecutionId) { + if (results.results && results.results.length > 0) { + await this.updateDetailedCaseResults(runId, resolvedExecutionId, results.results); + + // 修复12: 清理残留的 ERROR 占位符 + // 当 Jenkins 回调只返回部分测试结果时(如10个用例只返回3个), + // updateDetailedCaseResults 只会更新这3个用例,剩余的 ERROR 占位符不会被清理。 + // 需要根据整体执行状态批量更新这些残留占位符。 + await this.cleanupResidualErrorPlaceholders( + runId, + resolvedExecutionId, + results.results.length, + results + ); + } else { + await this.updateSummaryOnlyResults(runId, resolvedExecutionId, results); + } + } else { + logger.warn( + `Could not determine executionId for runId, skipping detailed result updates`, + { runId, resultsCount: results.results?.length ?? 0, cachedExecutionId: executionId }, + LOG_CONTEXTS.REPOSITORY + ); + + // 【兜底修复】resolvedExecutionId 无法从 TaskExecution 表获取时, + // 尝试直接从 Auto_TestRunResults 表通过已有占位记录反查 executionId, + // 确保 ERROR 占位符依然能被清理,避免用例状态永久卡在 error + const fallbackExecId = await this.resolveExecutionIdFromRunResults(runId); + logger.info( + `completeBatch: fallback executionId lookup result`, + { runId, fallbackExecId: fallbackExecId ?? null, found: !!fallbackExecId }, + LOG_CONTEXTS.REPOSITORY + ); + if (fallbackExecId) { + await this.updateSummaryOnlyResults(runId, fallbackExecId, results); + await this.performFinalErrorCleanup(fallbackExecId, results); + // 【修复3】兜底路径也要做最终一致性校正,确保 Auto_TestRun 统计与明细一致 + await this.reconcileBatchSummary(runId, fallbackExecId, mappedStatus); + } + } + + // 【安全防护】无论采用哪个更新路径,都执行最后的全局清理 + // 防止在特殊场景(既无详细结果也无汇总统计)下 ERROR 占位符残留 + if (resolvedExecutionId) { + await this.performFinalErrorCleanup(resolvedExecutionId, results); + // 最终一致性校正:以结果表为准回填批次汇总,避免"汇总 success 但明细仍 error" + await this.reconcileBatchSummary(runId, resolvedExecutionId, mappedStatus); + } + } + + /** + * 最终一致性校正:以 Auto_TestRunResults 结果表为准,回填 Auto_TestRun 和 Auto_TestCaseTaskExecutions 的汇总统计 + * 确保两张表数据一致,避免"汇总通过但明细失败"或相反的情况 + */ + private async reconcileBatchSummary( + runId: number, + executionId: number, + mappedStatus: 'pending' | 'running' | 'success' | 'failed' | 'aborted' + ): Promise { + const finalCounts = await this.countResultsByStatus(executionId); + if (finalCounts.total > 0) { + let reconciledRunStatus = mappedStatus; + if (finalCounts.failed > 0) { + reconciledRunStatus = 'failed'; + } else if (finalCounts.passed > 0) { + reconciledRunStatus = 'success'; + } + + await this.testRunRepository.update(runId, { + status: reconciledRunStatus, + passedCases: finalCounts.passed, + failedCases: finalCounts.failed, + skippedCases: finalCounts.skipped, + }); + + const reconciledTaskStatus: 'success' | 'failed' | 'cancelled' = + reconciledRunStatus === 'aborted' ? 'cancelled' + : reconciledRunStatus === 'success' ? 'success' + : 'failed'; + + await this.repository.update(executionId, { + status: reconciledTaskStatus, + passedCases: finalCounts.passed, + failedCases: finalCounts.failed, + skippedCases: finalCounts.skipped, + }); + + logger.info('Reconciled batch summary from result rows', { + runId, + executionId, + reconciledRunStatus, + finalCounts, + }, LOG_CONTEXTS.REPOSITORY); + } + } + + /** + * 【安全防护】最终的全局 ERROR 清理 + * 在所有其他更新完成后执行,确保不会有遗漏的 ERROR 占位符 + * 这是一个防御性的清理,不依赖任何前置条件 + */ + private async performFinalErrorCleanup(executionId: number, results: BatchResults): Promise { + try { + const errorRows = await this.testRunResultRepository.query(` + SELECT COUNT(*) AS errorCount + FROM Auto_TestRunResults + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + `, [executionId]) as Array<{ errorCount: string }>; + + const residualErrorCount = Number(errorRows[0]?.errorCount ?? 0); + + if (residualErrorCount === 0) { + logger.debug( + 'Final error cleanup: no orphaned errors found', + { executionId }, + LOG_CONTEXTS.REPOSITORY + ); + return; + } + + // 根据运行状态决定清理目标 + let targetStatus: 'passed' | 'failed' | 'skipped'; + const mappedStatus = this.mapStatusForTestRun(results.status); + + if (mappedStatus === 'success') { + targetStatus = 'passed'; + } else if (mappedStatus === 'failed') { + targetStatus = 'failed'; + } else { + targetStatus = 'skipped'; + } + + const cleaned = await this.bulkUpdateErrorResults(executionId, targetStatus); + + logger.info( + 'Final error cleanup completed', + { + executionId, + residualErrorCount, + cleaned, + targetStatus, + reason: 'safety-cleanup-after-all-updates', + }, + LOG_CONTEXTS.REPOSITORY + ); + } catch (error) { + logger.warn( + 'Final error cleanup failed, but execution will continue', + { + executionId, + error: error instanceof Error ? error.message : String(error), + }, + LOG_CONTEXTS.REPOSITORY + ); + } + } + + /** + * 【轮询路径清理】清理指定 execution 中的 ERROR 占位符 + * 用于轮询同步路径中,当 Jenkins 状态已完成但结果为虚拟/部分时,清理预创建的 ERROR 占位符 + */ + async cleanupErrorPlaceholdersForExecution(executionId: number, runStatus: string): Promise { + try { + // 查询是否存在 ERROR 占位符 + const errorRows = await this.testRunResultRepository.query(` + SELECT COUNT(*) AS errorCount + FROM Auto_TestRunResults + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + `, [executionId]) as Array<{ errorCount: string }>; + + const residualErrorCount = Number(errorRows[0]?.errorCount ?? 0); + + if (residualErrorCount === 0) { + logger.debug( + 'cleanupErrorPlaceholdersForExecution: no errors found', + { executionId }, + LOG_CONTEXTS.REPOSITORY + ); + return 0; + } + + // 根据运行状态映射清理目标 + let targetStatus: 'passed' | 'failed' | 'skipped'; + if (runStatus === 'success') { + targetStatus = 'passed'; + } else if (runStatus === 'failed') { + targetStatus = 'failed'; + } else { + targetStatus = 'skipped'; + } + + const cleaned = await this.bulkUpdateErrorResults(executionId, targetStatus); + + logger.info( + 'Cleaned error placeholders in polling sync path', + { + executionId, + runStatus, + residualErrorCount, + cleaned, + targetStatus, + reason: 'polling-sync-cleanup', + }, + LOG_CONTEXTS.REPOSITORY + ); + + return cleaned; + } catch (error) { + logger.warn( + 'Failed to cleanup error placeholders', + { + executionId, + error: error instanceof Error ? error.message : String(error), + }, + LOG_CONTEXTS.REPOSITORY + ); + return 0; + } + } + + /** + * 【紧急修复】修复孤立的 TestRun(没有绑定 execution_id) + * + * 问题:旧的 TestRun 没有设置 execution_id 字段,导致查询结果时通过时间窗口反查 + * 可能得到错误的 executionId,进而获取错误的用例结果 + * + * 解决方案: + * 1. 查找所有 execution_id 为 NULL 的 TestRun + * 2. 通过触发者 + 时间窗口匹配最近的 TaskExecution + * 3. 回填 execution_id 字段 + */ + + private async resolveExecutionIdForBatch(runId: number, cachedExecutionId?: number): Promise { + let runBoundExecutionId: number | undefined; + + // 优先读取 Auto_TestRun.execution_id,避免时间窗口误关联到其他执行 + try { + const trRows = await this.testRunRepository.query( + `SELECT execution_id FROM Auto_TestRun WHERE id = ? LIMIT 1`, + [runId] + ) as Array<{ execution_id: number | null }>; + if (trRows.length > 0 && trRows[0].execution_id) { + runBoundExecutionId = trRows[0].execution_id; + } + } catch (error) { + // 兼容旧库可能不存在 execution_id 列的场景 + logger.debug('Failed to read execution_id column, falling back to cache/time-window search', { + runId, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.REPOSITORY); + } + + if (runBoundExecutionId) { + if (cachedExecutionId && cachedExecutionId !== runBoundExecutionId) { + logger.warn('Cached executionId mismatch, using run-bound execution_id', { + runId, + cachedExecutionId, + runBoundExecutionId, + }, LOG_CONTEXTS.REPOSITORY); + } + logger.debug('Resolved executionId from Auto_TestRun.execution_id', { + runId, + resolvedId: runBoundExecutionId, + }, LOG_CONTEXTS.REPOSITORY); + return runBoundExecutionId; + } + + if (cachedExecutionId) { + logger.debug('Using cached executionId because run has no bound execution_id', { + runId, + cachedExecutionId, + }, LOG_CONTEXTS.REPOSITORY); + return cachedExecutionId; + } + + // 降级到时间窗口反查 + const fallbackId = await this.findExecutionIdByRunId(runId); + return fallbackId || undefined; + } + + /** + * 兜底策略:通过 Auto_TestRunResults 表中已有的占位记录反查 executionId + * + * 背景:triggerExecution 事务中会将 executionId 回填到 Auto_TestRun.execution_id, + * 同时也将 executionId 写入 Auto_TestRunResults 的每条预创建记录。 + * 若 Auto_TestRun.execution_id 字段 null(旧数据),时间窗口反查也失败时, + * 可以从 Auto_TestRunResults 里找到该 runId 对应的 executionId。 + * + * 实现:通过 Auto_TestRun.id 的创建时间找到在同时间段内插入的 TestRunResults, + * 从中读取 execution_id 作为 fallback。 + */ + protected async resolveExecutionIdFromRunResults(runId: number): Promise { + try { + // 先查该 runId 对应的 TestRun 创建时间 + const runRows = await this.testRunRepository.query( + `SELECT id, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1`, + [runId] + ) as Array<{ id: number; created_at: Date }>; + + if (!runRows || runRows.length === 0) return undefined; + + const createdAt = runRows[0].created_at; + + // 在 TestRun 创建时间 ±120 秒内,查找 Auto_TestRunResults 中存在的 execution_id + // triggerExecution 事务中同一批次的 TestRunResults 会在 TestRun 创建后立即插入 + const resultRows = await this.testRunResultRepository.query( + `SELECT DISTINCT execution_id + FROM Auto_TestRunResults + WHERE created_at BETWEEN DATE_SUB(?, INTERVAL 120 SECOND) + AND DATE_ADD(?, INTERVAL 120 SECOND) + AND execution_id IS NOT NULL + ORDER BY id ASC + LIMIT 1`, + [createdAt, createdAt] + ) as Array<{ execution_id: number }>; + + if (resultRows && resultRows.length > 0 && resultRows[0].execution_id) { + logger.debug('resolveExecutionIdFromRunResults: found executionId via TestRunResults time-window', { + runId, + executionId: resultRows[0].execution_id, + }, LOG_CONTEXTS.REPOSITORY); + return resultRows[0].execution_id; + } + + return undefined; + } catch (error) { + logger.debug('resolveExecutionIdFromRunResults: query failed', { + runId, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.REPOSITORY); + return undefined; + } + } + + /** + * 修复10: completeBatch 拆分 - 同步 TaskExecution(Auto_TestCaseTaskExecutions)状态 + * + * completeBatch 主要更新 Auto_TestRun,此方法负责将状态同步到 Auto_TestCaseTaskExecutions, + * 保证 getRecentExecutions 查询 TaskExecution 时也能读到最新状态。 + */ + private async syncTaskExecutionStatus(resolvedExecutionId: number, results: BatchResults): Promise { + // TaskExecution 不支持 'aborted',统一映射为 'cancelled' + const taskExecStatus: 'success' | 'failed' | 'cancelled' = + results.status === 'aborted' || results.status === 'cancelled' ? 'cancelled' + : results.status === 'success' ? 'success' + : 'failed'; + + await this.repository.update(resolvedExecutionId, { + status: taskExecStatus, + passedCases: results.passedCases, + failedCases: results.failedCases, + skippedCases: results.skippedCases, + duration: Math.round(results.durationMs / 1000), + endTime: new Date(), + }); + } + + /** + * 修复10: completeBatch 拆分 - 处理带详细用例结果的更新路径 + * + * 遍历 caseResults,逐条尝试更新已有记录;若记录不存在则新建。 + * 失败的条目会被收集并记录警告日志,但不会中断其余条目的处理。 + */ + private async updateDetailedCaseResults( + runId: number, + executionId: number, + caseResults: NonNullable + ): Promise { + const failedResults: Array<{ caseId?: number; error: string }> = []; + + for (const result of caseResults) { + try { + const resolveTime = (v: string | number | undefined): Date | undefined => { + if (!v) return undefined; + const d = new Date(v); + return isNaN(d.getTime()) ? undefined : d; + }; + const startTime = resolveTime(result.startTime) ?? new Date(); + const endTime = resolveTime(result.endTime) ?? new Date(); + + const normalizedStatus = this.normalizeCaseResultStatus(result.status); + + const updated = await this.updateTestResult(executionId, result.caseId, { + status: normalizedStatus, + duration: result.duration, + errorMessage: result.errorMessage, + errorStack: result.stackTrace, + screenshotPath: result.screenshotPath, + logPath: result.logPath, + assertionsTotal: result.assertionsTotal, + assertionsPassed: result.assertionsPassed, + responseData: result.responseData, + startTime, + endTime, + caseName: result.caseName, // 无 caseId 时按 caseName fallback 匹配 + }); + + // 【诊断日志】记录每条用例的匹配结果,便于追踪明细写入失败的原因 + logger.debug( + `updateDetailedCaseResults: case update result`, + { + executionId, + runId, + caseId: result.caseId, + caseName: result.caseName, + status: normalizedStatus, + matched: updated, + action: updated ? 'updated' : (result.caseId !== undefined ? 'will_create' : 'skipped_no_caseId'), + }, + LOG_CONTEXTS.REPOSITORY + ); + + if (!updated) { + // 若 caseId 缺失(caseName fallback 场景),跳过 createTestResult 避免 DB NOT NULL 错误 + if (result.caseId !== undefined) { + await this.createTestResult({ + executionId, + caseId: result.caseId, + caseName: result.caseName, + status: normalizedStatus, + duration: result.duration, + errorMessage: result.errorMessage, + errorStack: result.stackTrace, + screenshotPath: result.screenshotPath, + logPath: result.logPath, + assertionsTotal: result.assertionsTotal, + assertionsPassed: result.assertionsPassed, + responseData: result.responseData, + startTime, + endTime, + }); + } + } + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + failedResults.push({ caseId: result.caseId, error: errorMsg }); + logger.error( + `Failed to process result for case`, + { caseId: result.caseId, executionId, error: errorMsg }, + LOG_CONTEXTS.REPOSITORY + ); + } + } + + if (failedResults.length > 0) { + logger.warn( + `Some results failed to process in batch`, + { + runId, + failedCount: failedResults.length, + totalCount: caseResults.length, + failedCaseIds: failedResults.map(f => f.caseId), + }, + LOG_CONTEXTS.REPOSITORY + ); + } + } + + /** + * 修复12: 清理残留的 ERROR 占位符 + * + * 场景:Jenkins 回调只返回部分测试结果(如网络故障、超时、脚本异常) + * 例如:10个用例只返回3个结果,剩余7个 ERROR 占位符不会被 updateDetailedCaseResults 清理 + * + * 解决方案: + * 1. 统计已更新的结果数量 vs 回调中的统计数 (passedCases + failedCases + skippedCases) + * 2. 计算预期的总用例数(数据库记录总数) + * 3. 检查是否有残留的 ERROR 占位符 + * 4. 根据整体执行状态批量更新残留占位符 + * + * @param runId TestRun ID + * @param executionId TaskExecution ID + * @param detailedResultsCount 回调中详细结果的数量 + * @param results 回调中的完整结果数据 + */ + private async cleanupResidualErrorPlaceholders( + runId: number, + executionId: number, + detailedResultsCount: number, + results: BatchResults + ): Promise { + // 1. 获取该 executionId 下的所有结果状态分布 + const statusCounts = await this.countResultsByStatus(executionId); + const totalFromCallback = results.passedCases + results.failedCases + results.skippedCases; + + // 2. 检查是否有残留的 ERROR 占位符 + const errorCountQuery = await this.testRunResultRepository.query(` + SELECT COUNT(*) AS errorCount + FROM Auto_TestRunResults + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + `, [executionId]) as Array<{ errorCount: string }>; + const residualErrorCount = Number(errorCountQuery[0]?.errorCount ?? 0); + + // 如果没有残留 ERROR 占位符,无需处理 + if (residualErrorCount === 0) { + logger.debug( + `No residual ERROR placeholders found, skip cleanup`, + { runId, executionId, detailedResultsCount, totalFromCallback }, + LOG_CONTEXTS.REPOSITORY + ); + return; + } + + // 3. 判断是否有部分结果缺失的情况 + // 如果回调中的统计数与详细结果数不一致,说明确实存在部分结果缺失 + const hasPartialResults = detailedResultsCount < totalFromCallback || statusCounts.total > detailedResultsCount; + + logger.info( + `Detected residual ERROR placeholders, will clean up based on overall status`, + { + runId, + executionId, + detailedResultsCount, + totalFromCallback, + dbTotal: statusCounts.total, + residualErrorCount, + hasPartialResults, + overallStatus: results.status, + }, + LOG_CONTEXTS.REPOSITORY + ); + + // 4. 根据整体执行状态批量更新残留的 ERROR 占位符 + // 安全策略:当存在部分结果缺失时,不应用"假定性"状态,避免假阳性 + // - 如果整体状态是 success 且没有部分结果缺失,将残留 ERROR 更新为 passed + // - 如果整体状态是 success 但有部分结果缺失,将残留 ERROR 更新为 skipped(安全处理) + // - 如果整体状态是 failed,将残留 ERROR 更新为 failed(保守处理) + // - 如果整体状态是 aborted/cancelled,将残留 ERROR 更新为 skipped + let targetStatus: 'passed' | 'failed' | 'skipped'; + let reason: string; + + if (results.status === 'success') { + if (hasPartialResults) { + // 安全处理:部分结果缺失时,残留 ERROR 可能是未执行的用例,标记为 skipped 避免假阳性 + targetStatus = 'skipped'; + reason = 'overall execution succeeded but partial results missing - marking as skipped to avoid false positives'; + } else { + // 完整结果场景:残留 ERROR 可能是 Jenkins 未返回的结果,假设它们通过了 + targetStatus = 'passed'; + reason = 'overall execution succeeded with complete results'; + } + } else if (results.status === 'failed') { + // 失败场景:保守处理,将残留 ERROR 标记为 failed + targetStatus = 'failed'; + reason = 'overall execution failed'; + } else { + // 取消/中断场景:将残留 ERROR 标记为 skipped + targetStatus = 'skipped'; + reason = 'execution was cancelled or aborted'; + } + + const updatedCount = await this.bulkUpdateErrorResults(executionId, targetStatus); + + logger.info( + `Cleaned up residual ERROR placeholders`, + { + runId, + executionId, + residualErrorCount, + updatedCount, + targetStatus, + reason, + hasPartialResults, + overallStatus: results.status, + }, + LOG_CONTEXTS.REPOSITORY + ); + + // 5. 同步更新统计数(确保数据库统计与实际状态一致) + const newCounts = await this.countResultsByStatus(executionId); + await this.testRunRepository.update(runId, { + passedCases: newCounts.passed, + failedCases: newCounts.failed, + skippedCases: newCounts.skipped, + }); + } + + /** + * 修复10 + 修复11: completeBatch 拆分 - 处理只有汇总统计、没有详细用例结果的兜底路径 + * + * 策略一(totalSummary > 0):Jenkins 只传了统计数(passedCases/failedCases/skippedCases > 0), + * 按顺序将预创建的 error 记录批量更新为对应状态。 + * 修复11: 改为按状态分组使用 IN 批量 UPDATE,替代原来的逐条循环,减少 SQL 往返次数。 + * + * 策略二(totalSummary = 0):Jenkins 统计数全为0, + * 先查数据库现有结果,若已有真实结果则回填统计;若全是 error 记录则按整体状态批量更新。 + */ + private async updateSummaryOnlyResults( + runId: number, + executionId: number, + results: BatchResults + ): Promise { + const totalSummary = results.passedCases + results.failedCases + results.skippedCases; + + if (totalSummary > 0) { + logger.info( + `No detailed results provided, updating pre-created records using summary counts`, + { runId, executionId, passedCases: results.passedCases, failedCases: results.failedCases, skippedCases: results.skippedCases }, + LOG_CONTEXTS.REPOSITORY + ); + + // 【修复】只获取仍处于 error 状态的占位记录,避免覆盖已被 updateDetailedCaseResults 写入的真实结果 + // 原查询无 status 过滤,在并发/重试场景下会把已正确更新的记录重置为错误状态 + const preCreatedResults = await this.testRunResultRepository.query(` + SELECT id FROM Auto_TestRunResults + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + ORDER BY id ASC + `, [executionId]) as Array<{ id: number }>; + + const now = new Date(); + const batchUpdate = async (ids: number[], status: 'passed' | 'failed' | 'skipped') => { + if (ids.length === 0) return; + // 只更新 status 和 end_time,不填充 start_time 和 duration: + // 没有真实执行数据时,保留 NULL 让前端显示 "-" + const placeholders = ids.map(() => '?').join(', '); + await this.testRunResultRepository.query( + `UPDATE Auto_TestRunResults + SET status = ?, + end_time = COALESCE(end_time, ?) + WHERE id IN (${placeholders})`, + [status, now, ...ids] + ); + }; + + if (preCreatedResults.length > 0) { + const passedEnd = results.passedCases; + const failedEnd = passedEnd + results.failedCases; + + // 修复11: 按状态分组收集 id,再批量 UPDATE,替代原来的逐条循环 + const passedIds = preCreatedResults.slice(0, passedEnd).map(r => r.id); + const failedIds = preCreatedResults.slice(passedEnd, failedEnd).map(r => r.id); + const skippedIds = preCreatedResults.slice(failedEnd).map(r => r.id); + + logger.info( + `updateSummaryOnlyResults: distributing status to error placeholders`, + { + runId, + executionId, + totalErrorPlaceholders: preCreatedResults.length, + passedIds: passedIds.length, + failedIds: failedIds.length, + skippedIds: skippedIds.length, + summaryPassed: results.passedCases, + summaryFailed: results.failedCases, + summarySkipped: results.skippedCases, + }, + LOG_CONTEXTS.REPOSITORY + ); + + await Promise.all([ + batchUpdate(passedIds, 'passed'), + batchUpdate(failedIds, 'failed'), + batchUpdate(skippedIds, 'skipped'), + ]); + + logger.info( + `Updated pre-created result records using summary counts`, + { runId, executionId, updatedCount: preCreatedResults.length }, + LOG_CONTEXTS.REPOSITORY + ); + } else { + // 【修复】当 error 占位符已被轮询路径提前清理(改为 failed/passed)时, + // 若此次回调汇总数据与数据库明细记录不一致,需要重新修正以保持数据同步。 + // 场景:轮询路径在无 JUnit 结果时将占位符改为 failed,随后回调到达告知 passedCases>0。 + // 此时需要将 failed 记录中多余的(按顺序)重新分配为 passed。 + const currentCounts = await this.testRunResultRepository.query(` + SELECT + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed, + SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, + COUNT(*) AS total + FROM Auto_TestRunResults + WHERE execution_id = ? + `, [executionId]) as Array<{ passed: string; failed: string; skipped: string; total: string }>; + + if (currentCounts.length > 0) { + const dbPassed = Number(currentCounts[0].passed ?? 0); + const dbFailed = Number(currentCounts[0].failed ?? 0); + const dbTotal = Number(currentCounts[0].total ?? 0); + + const expectedPassed = results.passedCases; + const expectedFailed = results.failedCases; + + // 只在汇总数据与数据库明细不一致时进行修正(避免覆盖正确数据) + const needsReconcile = dbTotal > 0 && (dbPassed !== expectedPassed || dbFailed !== expectedFailed); + + if (needsReconcile) { + logger.info( + `updateSummaryOnlyResults: no error placeholders but counts mismatch, reconciling`, + { + runId, + executionId, + dbPassed, + dbFailed, + dbTotal, + expectedPassed, + expectedFailed, + summaryPassed: results.passedCases, + summaryFailed: results.failedCases, + }, + LOG_CONTEXTS.REPOSITORY + ); + + // 若回调说全部通过(passedCases == total),但 DB 里全是 failed, + // 则将所有非 passed 记录按序重新分配为 passed + if (expectedPassed > 0 && dbPassed < expectedPassed) { + // 查出多余的 failed 记录,按顺序将前 (expectedPassed - dbPassed) 条改为 passed + const needMorePassed = expectedPassed - dbPassed; + const failedRecords = await this.testRunResultRepository.query(` + SELECT id FROM Auto_TestRunResults + WHERE execution_id = ? AND status = 'failed' + ORDER BY id ASC + LIMIT ? + `, [executionId, needMorePassed]) as Array<{ id: number }>; + + if (failedRecords.length > 0) { + const ids = failedRecords.map(r => r.id); + await batchUpdate(ids, 'passed'); + logger.info( + `updateSummaryOnlyResults: reconciled failed→passed records`, + { runId, executionId, reconciled: ids.length, expectedPassed, dbPassed }, + LOG_CONTEXTS.REPOSITORY + ); + } + } else if (expectedFailed > 0 && dbFailed < expectedFailed) { + // 反向场景:DB 里 passed 多,但 Jenkins 说有 failed + const needMoreFailed = expectedFailed - dbFailed; + const passedRecords = await this.testRunResultRepository.query(` + SELECT id FROM Auto_TestRunResults + WHERE execution_id = ? AND status = 'passed' + ORDER BY id DESC + LIMIT ? + `, [executionId, needMoreFailed]) as Array<{ id: number }>; + + if (passedRecords.length > 0) { + const ids = passedRecords.map(r => r.id); + await batchUpdate(ids, 'failed'); + logger.info( + `updateSummaryOnlyResults: reconciled passed→failed records`, + { runId, executionId, reconciled: ids.length, expectedFailed, dbFailed }, + LOG_CONTEXTS.REPOSITORY + ); + } + } + } + } + } + } else { + // 统计数全为0,且没有详细 results 数组 + // 先查数据库里该 executionId 下的现有记录状态 + const countRows = await this.testRunResultRepository.query(` + SELECT + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed, + SUM(CASE WHEN status IS NULL OR status = 'error' THEN 1 ELSE 0 END) AS error_count, + SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, + COUNT(*) AS total + FROM Auto_TestRunResults + WHERE execution_id = ? + `, [executionId]) as Array<{ passed: string; failed: string; error_count: string; skipped: string; total: string }>; + + if (countRows.length > 0) { + const dbPassed = Number(countRows[0].passed ?? 0); + const dbFailed = Number(countRows[0].failed ?? 0); + const dbError = Number(countRows[0].error_count ?? 0); + const dbSkipped = Number(countRows[0].skipped ?? 0); + const dbTotal = Number(countRows[0].total ?? 0); + const dbFinished = dbPassed + dbFailed + dbSkipped; + + if (dbTotal > 0 && dbFinished > 0) { + // 已有非 error 的真实结果,直接用统计值回填 Auto_TestRun,不覆盖为0 + await this.testRunRepository.update(runId, { + passedCases: dbPassed, + failedCases: dbFailed, + skippedCases: dbSkipped, + }); + logger.info( + `Recalculated stats from existing results and back-filled Auto_TestRun`, + { runId, executionId, dbPassed, dbFailed, dbSkipped }, + LOG_CONTEXTS.REPOSITORY + ); + } else if (dbTotal > 0 && dbError > 0) { + // 全是预创建的 error 记录,根据执行整体状态将它们批量更新 + // success → passed;failed/aborted/cancelled → failed + const mappedStatus: 'passed' | 'failed' = + (results.status === 'success') ? 'passed' : 'failed'; + + // 只更新 status 和 end_time,不填充 start_time 和 duration,保留 NULL 让前端正确显示 "-" + await this.testRunResultRepository.query(` + UPDATE Auto_TestRunResults + SET status = ?, + end_time = COALESCE(end_time, NOW()) + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + `, [mappedStatus, executionId]); + + // 根据映射结果更新 Auto_TestRun 统计 + const newPassed = mappedStatus === 'passed' ? dbError : 0; + const newFailed = mappedStatus === 'failed' ? dbError : 0; + await this.testRunRepository.update(runId, { + passedCases: newPassed, + failedCases: newFailed, + skippedCases: 0, + }); + + logger.info( + `Bulk-updated pre-created error records based on overall execution status`, + { runId, executionId, mappedStatus, updatedCount: dbError, newPassed, newFailed }, + LOG_CONTEXTS.REPOSITORY + ); + } + } + } + } +} diff --git a/server/repositories/ExecutionRepositoryLookup.ts b/server/repositories/ExecutionRepositoryLookup.ts new file mode 100644 index 0000000..7997e9d --- /dev/null +++ b/server/repositories/ExecutionRepositoryLookup.ts @@ -0,0 +1,518 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; +import { ExecutionRepositoryBase } from './ExecutionRepositoryBase'; +import type { + BatchResults, + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; +export abstract class ExecutionRepositoryLookup extends ExecutionRepositoryBase { + protected abstract resolveExecutionIdFromRunResults(runId: number): Promise; + + async findExecutionId(): Promise { + const result = await this.testRunResultRepository.createQueryBuilder('result') + .select('DISTINCT result.executionId') + .where('result.executionId IS NOT NULL') + .orderBy('result.executionId', 'DESC') + .limit(1) + .getRawOne(); + + return result?.executionId || null; + } + + /** + * 根据 runId 查找关联的 executionId + * + * 设计说明: + * - Auto_TestRun 和 Auto_TestCaseTaskExecutions 同时创建(时间相近) + * - Auto_TestRunResults 的 execution_id 指向 Auto_TestCaseTaskExecutions.id + * - 通过 Auto_TestCaseTaskExecutions 表,用触发者 + 时间窗口反查 executionId + * + * 查询策略: + * 1. 获取 TestRun 的创建时间和触发者信息 + * 2. 通过时间窗口(±TIME_WINDOW_TOLERANCE_SECONDS)+ 触发者匹配查找最近的 TaskExecution + * 3. 如果没有找到结果,记录警告并返回 null + * + * @param runId 执行批次ID + * @returns 关联的运行记录ID,如果找不到则返回 null + */ + async findExecutionIdByRunId(runId: number): Promise { + // 1. 获取 TestRun 的详细信息(含创建时间) + const testRunRows = await this.testRunRepository.query(` + SELECT id, trigger_by, created_at FROM Auto_TestRun WHERE id = ? LIMIT 1 + `, [runId]) as Array<{ id: number; trigger_by: number; created_at: Date }>; + + if (!testRunRows || testRunRows.length === 0) { + logger.warn( + `TestRun not found`, + { runId }, + LOG_CONTEXTS.REPOSITORY + ); + return null; + } + + const testRun = testRunRows[0]; + + // 2. 通过时间窗口(±TIME_WINDOW_TOLERANCE_SECONDS)和触发者信息,在 Auto_TestCaseTaskExecutions 中查找最近的关联记录 + // 使用时间窗口而非 id 差值,避免因两表 id 自增不同步导致查找失败 + // 查询逻辑: + // - 匹配同一触发者(executed_by) + // - 创建时间在 TestRun 前后 TIME_WINDOW_TOLERANCE_SECONDS 秒内 + // - 按时间差绝对值升序排列,取最近的记录 + const result = await this.testRunResultRepository.query(` + SELECT e.id as execution_id + FROM Auto_TestCaseTaskExecutions e + WHERE e.executed_by = ? + AND e.created_at BETWEEN DATE_SUB(?, INTERVAL ? SECOND) AND DATE_ADD(?, INTERVAL ? SECOND) + ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC + LIMIT 1 + `, [ + testRun.trigger_by, + testRun.created_at, ExecutionRepositoryBase.TIME_WINDOW_TOLERANCE_SECONDS, + testRun.created_at, ExecutionRepositoryBase.TIME_WINDOW_TOLERANCE_SECONDS, + testRun.created_at, + ]); + + if (result && result.length > 0 && result[0].execution_id) { + logger.debug( + `Found executionId for runId via time-window`, + { runId, executionId: result[0].execution_id }, + LOG_CONTEXTS.REPOSITORY + ); + return result[0].execution_id; + } + + // 3. 尝试通过 TestRunResults 表反查 executionId + const fallbackExecutionId = await this.resolveExecutionIdFromRunResults(runId); + if (fallbackExecutionId) { + logger.info( + `Found executionId for runId via TestRunResults fallback`, + { runId, executionId: fallbackExecutionId }, + LOG_CONTEXTS.REPOSITORY + ); + return fallbackExecutionId; + } + + // 4. 如果没有找到,记录警告 + logger.warn( + `Could not find executionId for runId`, + { + runId, + triggerBy: testRun.trigger_by, + suggestion: 'Consider adding execution_id column to Auto_TestRun table' + }, + LOG_CONTEXTS.REPOSITORY + ); + + return null; + } + + /** + * 更新测试结果 + * 修复4: 使用 TestRunResultStatus 枚举校验状态,替代不安全的强制类型断言 + */ + async updateTestResult( + executionId: number, + caseId: number | undefined | null, + result: { + status: string; + duration: number; + errorMessage?: string; + errorStack?: string; + screenshotPath?: string; + logPath?: string; + assertionsTotal?: number; + assertionsPassed?: number; + responseData?: string; + startTime?: Date; + endTime?: Date; + caseName?: string; + } + ): Promise { + // 将外部传入的 status 字符串映射为合法的枚举值,不合法时降级为 'error' + const validStatuses: ReadonlyArray = Object.values(TestRunResultStatus); + const safeStatus = validStatuses.includes(result.status) + ? (result.status as 'passed' | 'failed' | 'skipped' | 'error') + : 'error'; + + const updateData: Partial = { + status: safeStatus, + duration: result.duration, + errorMessage: result.errorMessage || null, + errorStack: result.errorStack || null, + screenshotPath: result.screenshotPath || null, + logPath: result.logPath || null, + assertionsTotal: result.assertionsTotal || null, + assertionsPassed: result.assertionsPassed || null, + responseData: result.responseData || null, + startTime: result.startTime || null, + endTime: result.endTime || null, + }; + + // 【修复】优先用 caseId 匹配;无 caseId 或为 0 时降级用 caseName 匹配(Jenkins 只传 caseName 的场景) + if (caseId && caseId > 0) { + const updateResult = await this.testRunResultRepository.update( + { executionId, caseId }, + updateData + ); + if ((updateResult.affected ?? 0) > 0) return true; + } + + // 【修复】当 caseId 缺失或为 0 时,尝试用 caseName 匹配 + // 优先精确匹配,其次模糊匹配(以防格式略有差异) + if (result.caseName) { + // 【第 2 层】精确匹配(完全相同) + const updateResult = await this.testRunResultRepository + .createQueryBuilder() + .update(TestRunResult) + .set(updateData) + .where('execution_id = :executionId AND case_name = :caseName', { + executionId, + caseName: result.caseName, + }) + .execute(); + if ((updateResult.affected ?? 0) > 0) return true; + + // 【第 3 层】大小写不敏感的精确匹配 + const caseInsensitiveResult = await this.testRunResultRepository + .createQueryBuilder() + .update(TestRunResult) + .set(updateData) + .where('execution_id = :executionId AND LOWER(case_name) = LOWER(:caseName)', { + executionId, + caseName: result.caseName, + }) + .execute(); + if ((caseInsensitiveResult.affected ?? 0) > 0) return true; + + // 【第 4 层】包含式模糊匹配(占位符 caseName 包含回调的 caseName) + // 例:期望 'TestGeolocation::test_geolocation' 但收到 'test_geolocation' + const fuzzyUpdateResult = await this.testRunResultRepository + .createQueryBuilder() + .update(TestRunResult) + .set(updateData) + .where('execution_id = :executionId AND (LOWER(case_name) LIKE LOWER(:fuzzyPattern1) OR LOWER(case_name) LIKE LOWER(:fuzzyPattern2))', { + executionId, + fuzzyPattern1: `%${result.caseName}%`, + fuzzyPattern2: `%${result.caseName.replace(/.*::/g, '')}%`, // 去掉命名空间 + }) + .execute(); + if ((fuzzyUpdateResult.affected ?? 0) > 0) return true; + } + + return false; + } + + /** + * 创建测试结果记录 + * 修复4: 使用 TestRunResultStatus 枚举校验状态,替代不安全的强制类型断言 + */ + async createTestResult(result: { + executionId: number; + caseId: number; + caseName: string; + status: string; + duration: number; + errorMessage?: string; + errorStack?: string; + screenshotPath?: string; + logPath?: string; + assertionsTotal?: number; + assertionsPassed?: number; + responseData?: string; + startTime?: Date; + endTime?: Date; + }): Promise { + // 将外部传入的 status 字符串映射为合法的枚举值,不合法时降级为 'error' + const validStatuses: ReadonlyArray = Object.values(TestRunResultStatus); + const safeStatus = validStatuses.includes(result.status) + ? (result.status as 'passed' | 'failed' | 'skipped' | 'error') + : 'error'; + + const entity = this.testRunResultRepository.create({ + executionId: result.executionId, + caseId: result.caseId, + caseName: result.caseName, + status: safeStatus, + duration: result.duration, + errorMessage: result.errorMessage || null, + errorStack: result.errorStack || null, + screenshotPath: result.screenshotPath || null, + logPath: result.logPath || null, + assertionsTotal: result.assertionsTotal || null, + assertionsPassed: result.assertionsPassed || null, + responseData: result.responseData || null, + startTime: result.startTime || null, + endTime: result.endTime || null, + }); + await this.testRunResultRepository.save(entity); + } + + /** + * 标记执行为超时 + */ + async markExecutionAsTimedOut(runId: number): Promise { + await this.testRunRepository.update(runId, { + status: TestRunStatus.ABORTED, + endTime: new Date(), + }); + } + + /** + * 获取可能超时的运行记录 + */ + async getPotentiallyTimedOutExecutions(timeoutThreshold: Date): Promise { + return this.testRunRepository.createQueryBuilder('testRun') + .select([ + 'testRun.id', + 'testRun.jenkinsJob', + 'testRun.jenkinsBuildId', + 'testRun.startTime', + ]) + .where('testRun.status IN (:...statuses)', { statuses: [TestRunStatus.PENDING, TestRunStatus.RUNNING] }) + .andWhere('testRun.startTime < :timeoutThreshold', { timeoutThreshold }) + .getRawMany(); + } + + /** + * 获取有 Jenkins 信息的运行记录 + */ + async getExecutionsWithJenkinsInfo(limit: number = 50): Promise { + return this.testRunRepository.createQueryBuilder('testRun') + .select([ + 'testRun.id', + 'testRun.status', + 'testRun.jenkinsJob', + 'testRun.jenkinsBuildId', + ]) + .where('testRun.jenkinsJob IS NOT NULL') + .andWhere('testRun.jenkinsBuildId IS NOT NULL') + .orderBy('testRun.id', 'DESC') + .limit(limit) + .getRawMany(); + } + + /** + * 获取可能卡住的运行记录(用于 ExecutionMonitorService) + * 查询状态为 pending/running 且超过指定时间阈值的运行记录 + */ + async getPotentiallyStuckExecutions(thresholdSeconds: number, limit: number = 20): Promise { + // 只检查最近 N 小时内的执行(优化:避免查询过期的旧执行) + // 从环境变量读取配置,默认 24 小时 + const maxAgeHours = parseInt(process.env.EXECUTION_MONITOR_MAX_AGE_HOURS || '24', 10); + + return this.testRunRepository.createQueryBuilder('testRun') + .select([ + 'testRun.id as id', + 'testRun.status as status', + 'testRun.jenkinsJob as jenkinsJob', + 'testRun.jenkinsBuildId as jenkinsBuildId', + 'testRun.startTime as startTime', + 'TIMESTAMPDIFF(SECOND, testRun.startTime, NOW()) as durationSeconds', + ]) + .where('testRun.status IN (:...statuses)', { statuses: [TestRunStatus.PENDING, TestRunStatus.RUNNING] }) + .andWhere('testRun.startTime IS NOT NULL') + .andWhere('TIMESTAMPDIFF(SECOND, testRun.startTime, NOW()) > :thresholdSeconds', { thresholdSeconds }) + // 只检查最近 N 小时内启动的执行(避免查询过期执行,用 start_time 代替 created_at) + .andWhere('testRun.startTime > DATE_SUB(NOW(), INTERVAL :maxAgeHours HOUR)', { maxAgeHours }) + .orderBy('testRun.startTime', 'ASC') + .limit(limit) + .getRawMany(); + } + + /** + * 获取测试运行的基本信息 + */ + async getTestRunBasicInfo(runId: number): Promise { + return this.testRunRepository.findOne({ + where: { id: runId }, + select: ['totalCases'], + }); + } + + /** + * 标记超时的旧执行为 aborted(清理过期卡住的执行) + * 覆盖两类僵尸记录: + * 1. start_time 不为 null 且超过 maxAgeHours(正常超时的 running/pending) + * 2. start_time IS NULL 且 created_at 超过 stuckPendingMinutes(Jenkins 从未触发的 pending,服务重启时队列丢失) + * @param maxAgeHours 最大运行时长(小时),超过时清理 start_time 不为 null 的记录 + * @param stuckPendingMinutes Jenkins 未触发的 pending 最长保留时间(分钟),默认 10 分钟 + * @returns 更新的执行数量 + */ + async markOldStuckExecutionsAsAbandoned(maxAgeHours: number = 24, stuckPendingMinutes: number = 10): Promise { + // 使用原生 SQL 支持 OR 条件,避免 TypeORM QueryBuilder 的 OR 限制 + const result = await this.testRunRepository.query( + `UPDATE Auto_TestRun + SET status = 'aborted', end_time = NOW() + WHERE status IN ('pending', 'running') + AND ( + -- 类型1:已开始但运行超时(start_time 不为 null) + (start_time IS NOT NULL AND start_time < DATE_SUB(NOW(), INTERVAL ? HOUR)) + OR + -- 类型2:Jenkins 从未触发(start_time 为 null),创建超过 N 分钟的 pending 记录 + (start_time IS NULL AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE)) + )`, + [maxAgeHours, stuckPendingMinutes] + ) as { affectedRows?: number; changedRows?: number }; + + return result?.affectedRows ?? 0; + } + + /** + * 汇总历史卡住记录(用于运行记录页提示条) + */ + async getStaleExecutionSummary(maxAgeHours: number = 24, stuckPendingMinutes: number = 10): Promise { + const rows = await this.testRunRepository.query( + `SELECT + SUM( + CASE + WHEN status = 'pending' + AND start_time IS NULL + AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE) + THEN 1 ELSE 0 + END + ) AS stale_pending_no_start_count, + SUM( + CASE + WHEN status IN ('pending', 'running') + AND start_time IS NOT NULL + AND start_time < DATE_SUB(NOW(), INTERVAL ? HOUR) + THEN 1 ELSE 0 + END + ) AS stale_started_count, + MAX( + CASE + WHEN status = 'pending' + AND start_time IS NULL + AND created_at < DATE_SUB(NOW(), INTERVAL ? MINUTE) + THEN created_at ELSE NULL + END + ) AS latest_stale_pending_created_at + FROM Auto_TestRun + WHERE status IN ('pending', 'running')`, + [stuckPendingMinutes, maxAgeHours, stuckPendingMinutes] + ) as Array<{ + stale_pending_no_start_count: number | string | null; + stale_started_count: number | string | null; + latest_stale_pending_created_at: Date | string | null; + }>; + + const row = rows[0] ?? { + stale_pending_no_start_count: 0, + stale_started_count: 0, + latest_stale_pending_created_at: null, + }; + + const stalePendingNoStartCount = Number(row.stale_pending_no_start_count ?? 0); + const staleStartedCount = Number(row.stale_started_count ?? 0); + + return { + stalePendingNoStartCount, + staleStartedCount, + totalStaleCount: stalePendingNoStartCount + staleStartedCount, + latestStalePendingCreatedAt: row.latest_stale_pending_created_at ? new Date(row.latest_stale_pending_created_at) : null, + }; + } + + /** + * 完整的触发测试执行流程(包含事务) + */ + async triggerExecution(input: { + caseIds: number[]; + projectId: number; + /** null 表示系统调度触发(无操作人) */ + triggeredBy: number | null; + triggerType: 'manual' | 'jenkins' | 'schedule'; + jenkinsJob?: string; + runConfig?: Record; + taskId?: number; + taskName?: string; + }): Promise<{ runId: number; executionId: number; totalCases: number; caseIds: number[] }> { + return this.executeInTransaction(async (_queryRunner) => { + // 1. 获取活跃用例 + const cases = await this.testCaseRepository.find({ + where: { + id: In(input.caseIds), + enabled: true, + }, + select: ['id', 'name', 'type', 'scriptPath'], + }); + + if (!cases || cases.length === 0) { + throw new Error(`No active test cases found with IDs: ${input.caseIds.join(',')}`); + } + + // 2. 创建测试运行记录 + const testRun = await this.createTestRun({ + projectId: input.projectId, + triggerType: input.triggerType, + triggerBy: input.triggeredBy, + jenkinsJob: input.jenkinsJob, + runConfig: input.runConfig, + totalCases: cases.length, + }); + + // 3. 创建任务运行记录 + const taskExecution = await this.createTaskExecution({ + taskId: input.taskId, + taskName: input.taskName, + totalCases: cases.length, + executedBy: input.triggeredBy, + }); + + // 4. 回填 executionId 到 TestRun(直接关联,消除时间窗口反查依赖) + await this.testRunRepository.update(testRun.id, { executionId: taskExecution.id }); + + // 5. 批量创建测试结果记录(初始状态为 error,等待 Jenkins 回调更新) + const testResults = cases.map(testCase => ({ + executionId: taskExecution.id, + caseId: testCase.id, + caseName: testCase.name, + status: null, + })); + + await this.createTestResults(testResults); + + return { + runId: testRun.id, + executionId: taskExecution.id, + totalCases: cases.length, + caseIds: cases.map(c => c.id), + }; + }); + } + + /** + * 完成批次执行 + * + * 修复10: 将原来 281 行的大方法拆分为若干职责明确的私有方法: + * - resolveExecutionIdForBatch: 多策略解析 executionId + * - syncTaskExecutionStatus: 同步 TaskExecution 状态 + * - updateDetailedCaseResults: 处理带详细结果的更新路径 + * - updateSummaryOnlyResults: 处理只有汇总统计的兜底路径 + * + * @param runId 执行批次ID + * @param results 执行结果 + * @param executionId 可选的执行ID(来自缓存,用于优化) + */ +} diff --git a/server/repositories/ExecutionRepositoryMaintenance.ts b/server/repositories/ExecutionRepositoryMaintenance.ts new file mode 100644 index 0000000..15b795d --- /dev/null +++ b/server/repositories/ExecutionRepositoryMaintenance.ts @@ -0,0 +1,309 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; +import { ExecutionRepositoryBase } from './ExecutionRepositoryBase'; +import { ExecutionRepositoryBatch } from './ExecutionRepositoryBatch'; +import type { + BatchResults, + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; +export class ExecutionRepositoryMaintenance extends ExecutionRepositoryBatch { + async fixOrphanedTestRuns(): Promise<{ fixed: number; checked: number }> { + let fixed = 0; + let checked = 0; + + try { + // 查找所有 execution_id 为 NULL 的 TestRun + const orphanedRuns = await this.testRunRepository.query(` + SELECT tr.id, tr.trigger_by, tr.created_at + FROM Auto_TestRun tr + WHERE tr.execution_id IS NULL + ORDER BY tr.id DESC + LIMIT 100 + `) as Array<{ id: number; trigger_by: number; created_at: Date }>; + + checked = orphanedRuns.length; + + if (orphanedRuns.length === 0) { + logger.info('No orphaned TestRuns found', {}, LOG_CONTEXTS.REPOSITORY); + return { fixed: 0, checked: 0 }; + } + + logger.info(`Found ${orphanedRuns.length} orphaned TestRuns, attempting to fix...`, {}, LOG_CONTEXTS.REPOSITORY); + + for (const run of orphanedRuns) { + // 通过时间窗口 + 触发者查找最近的 TaskExecution + const matchingExecutions = await this.repository.query(` + SELECT e.id as execution_id + FROM Auto_TestCaseTaskExecutions e + WHERE e.executed_by = ? + AND e.created_at BETWEEN DATE_SUB(?, INTERVAL 120 SECOND) AND DATE_ADD(?, INTERVAL 120 SECOND) + ORDER BY ABS(TIMESTAMPDIFF(SECOND, e.created_at, ?)) ASC + LIMIT 1 + `, [ + run.trigger_by, + run.created_at, run.created_at, run.created_at + ]) as Array<{ execution_id: number }>; + + if (matchingExecutions && matchingExecutions.length > 0) { + const executionId = matchingExecutions[0].execution_id; + await this.testRunRepository.update(run.id, { executionId }); + fixed++; + logger.info(`Fixed TestRun #${run.id} → executionId ${executionId}`, {}, LOG_CONTEXTS.REPOSITORY); + } + } + + logger.info(`Orphaned TestRuns fix completed: ${fixed}/${checked} fixed`, { fixed, checked }, LOG_CONTEXTS.REPOSITORY); + } catch (error) { + logger.warn( + 'Failed to fix orphaned TestRuns', + { error: error instanceof Error ? error.message : String(error) }, + LOG_CONTEXTS.REPOSITORY + ); + } + + return { fixed, checked }; + } + + /** + * 更新测试运行的状态和 Jenkins 信息(用于 Jenkins 同步) + */ + async updateTestRunStatus( + runId: number, + status: string, + options?: { + durationMs?: number; + passedCases?: number; + failedCases?: number; + skippedCases?: number; + abortReason?: string; + } + ): Promise { + const normalizedStatus = status === 'cancelled' ? 'aborted' : status; + const updateData: QueryDeepPartialEntity = { + status: normalizedStatus as 'pending' | 'running' | 'success' | 'failed' | 'aborted', + }; + + // 终态时设置 endTime + if (['success', 'failed', 'aborted'].includes(normalizedStatus)) { + updateData.endTime = new Date(); + } else if (['running', 'pending'].includes(normalizedStatus)) { + // 非终态(running/pending)时清除 endTime,防止从终态回退时出现数据不一致 + // 场景:Jenkins 轮询误判(building=true)或网络延迟导致状态错乱 + // 确保数据一致性:running/pending 状态不应有结束时间 + updateData.endTime = null as unknown as Date; + } + + if (options?.durationMs !== undefined) updateData.durationMs = options.durationMs; + if (options?.passedCases !== undefined) updateData.passedCases = options.passedCases; + if (options?.failedCases !== undefined) updateData.failedCases = options.failedCases; + if (options?.skippedCases !== undefined) updateData.skippedCases = options.skippedCases; + + if (normalizedStatus === 'aborted' && options?.abortReason) { + const current = await this.testRunRepository.findOne({ + where: { id: runId }, + select: ['id', 'runConfig'], + }); + const currentConfig = current?.runConfig; + let parsedConfig: Record = {}; + + if (currentConfig && typeof currentConfig === 'object' && !Array.isArray(currentConfig)) { + parsedConfig = currentConfig as Record; + } else if (typeof currentConfig === 'string') { + try { + const parsed = JSON.parse(currentConfig) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + parsedConfig = parsed as Record; + } + } catch { + parsedConfig = {}; + } + } + + updateData.runConfig = { + ...parsedConfig, + abortReason: options.abortReason, + abortAt: new Date().toISOString(), + }; + } + + await this.testRunRepository.update(runId, updateData); + } + + /** + * 获取测试运行状态信息 + */ + async getTestRunStatus(runId: number): Promise { + return this.testRunRepository.findOne({ + where: { id: runId }, + select: ['id', 'executionId', 'status', 'jenkinsJob', 'jenkinsBuildId', 'jenkinsUrl', 'startTime'], + }); + } + + async syncTaskExecutionFromTestRunStatus( + runId: number, + status: string, + options?: { + durationMs?: number; + passedCases?: number; + failedCases?: number; + skippedCases?: number; + } + ): Promise { + const testRun = await this.testRunRepository.findOne({ + where: { id: runId }, + select: ['id', 'executionId', 'passedCases', 'failedCases', 'skippedCases', 'durationMs'], + }); + + if (!testRun?.executionId) { + logger.warn('syncTaskExecutionFromTestRunStatus: no executionId bound to run', { + runId, + }, LOG_CONTEXTS.REPOSITORY); + return; + } + + const currentTaskExecution = await this.repository.findOne({ + where: { id: testRun.executionId }, + select: ['id', 'startTime', 'endTime'], + }); + + const normalizedStatus: 'pending' | 'running' | 'success' | 'failed' | 'cancelled' = + status === 'aborted' || status === 'cancelled' ? 'cancelled' + : status === 'success' ? 'success' + : status === 'running' ? 'running' + : status === 'pending' ? 'pending' + : 'failed'; + + const updateData: QueryDeepPartialEntity = { + status: normalizedStatus, + passedCases: options?.passedCases ?? testRun.passedCases ?? 0, + failedCases: options?.failedCases ?? testRun.failedCases ?? 0, + skippedCases: options?.skippedCases ?? testRun.skippedCases ?? 0, + duration: Math.round((options?.durationMs ?? testRun.durationMs ?? 0) / 1000), + }; + + if (normalizedStatus === 'running') { + updateData.startTime = currentTaskExecution?.startTime ?? new Date(); + updateData.endTime = null as unknown as Date; + } else if (['success', 'failed', 'cancelled'].includes(normalizedStatus)) { + updateData.startTime = currentTaskExecution?.startTime ?? new Date(); + updateData.endTime = currentTaskExecution?.endTime ?? new Date(); + } + + await this.repository.update(testRun.executionId, updateData); + } + + /** + * [dev-11] 获取当前所有 running 状态的执行记录(用于服务启动时恢复调度器槽位) + * 只查最近 maxAgeHours 小时内启动的执行,避免捞出陈年旧账 + * 使用原生 SQL 避免 TypeORM 实体元数据依赖(防止启动顺序竞态问题) + * 修复12: 添加 ACTIVE_SLOTS_MAX_LIMIT 上限,防止异常场景下返回过多记录 + */ + async getActiveRunningSlots(maxAgeHours: number = 24): Promise> { + const rows = await this.testRunRepository.query( + `SELECT r.id, e.task_id AS taskId, r.start_time AS startTime + FROM Auto_TestRun r + LEFT JOIN Auto_TestCaseTaskExecutions e ON r.execution_id = e.id + WHERE r.status = 'running' + AND r.start_time > DATE_SUB(NOW(), INTERVAL ? HOUR) + ORDER BY r.start_time ASC + LIMIT ?`, + [maxAgeHours, ExecutionRepositoryBase.ACTIVE_SLOTS_MAX_LIMIT] + ) as Array<{ id: number; taskId: number | null; startTime: Date | null }>; + return rows; + } + + /** + * 将指定 executionId 下所有 status=error 的预创建记录批量更新为目标状态 + * 同时填充 start_time(若为 NULL 则用 NOW())和 duration(若为 NULL 则用 0) + */ + async bulkUpdateErrorResults(executionId: number, targetStatus: 'passed' | 'failed' | 'skipped'): Promise { + // 只更新 status 和 end_time,不填充 start_time 和 duration: + // 这些占位符记录没有真实的执行时间和耗时数据,保留 NULL 让前端显示 "-", + // 避免用当前时间或 0 误导用户。 + const result = await this.testRunResultRepository.query(` + UPDATE Auto_TestRunResults + SET status = ?, + end_time = COALESCE(end_time, NOW()) + WHERE execution_id = ? AND (status IS NULL OR status = 'error') + `, [targetStatus, executionId]) as { affectedRows?: number; changedRows?: number }; + return result?.affectedRows ?? 0; + } + + /** + * 统计指定 executionId 下各状态的结果数量 + */ + + async syncTestRunByExecutionId(executionId: number, data: { + status: 'success' | 'failed' | 'cancelled' | 'aborted'; + passedCases: number; + failedCases: number; + skippedCases: number; + durationMs: number; + }): Promise { + // Auto_TestRun.execution_id 字段直接关联了 executionId + const testRun = await this.testRunRepository.findOne({ + where: { executionId }, + select: ['id', 'status'], + }); + + if (!testRun) { + logger.warn('syncTestRunByExecutionId: no TestRun found for executionId', { executionId }, LOG_CONTEXTS.REPOSITORY); + return false; + } + + const mappedStatus = this.mapStatusForTestRun(data.status); + await this.testRunRepository.update(testRun.id, { + status: mappedStatus, + passedCases: data.passedCases, + failedCases: data.failedCases, + skippedCases: data.skippedCases, + durationMs: data.durationMs, + endTime: new Date(), + }); + + logger.debug('syncTestRunByExecutionId: synced TestRun stats', { + executionId, + runId: testRun.id, + passedCases: data.passedCases, + failedCases: data.failedCases, + skippedCases: data.skippedCases, + }, LOG_CONTEXTS.REPOSITORY); + + return true; + } + + // ============================================================================ + // 私有辅助方法 + // ============================================================================ + + /** + * 辅助方法:将状态映射为 TestRun 的枚举值 + * @param status 输入状态 + * @returns TestRun 的状态枚举值 + */ +} diff --git a/server/repositories/ExecutionRepositoryStatusUtilities.ts b/server/repositories/ExecutionRepositoryStatusUtilities.ts new file mode 100644 index 0000000..fd17b31 --- /dev/null +++ b/server/repositories/ExecutionRepositoryStatusUtilities.ts @@ -0,0 +1,101 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; +import { ExecutionRepositoryLookup } from './ExecutionRepositoryLookup'; +import { ExecutionRepositoryBase } from './ExecutionRepositoryBase'; +import type { + BatchResults, + ExecutionDetail, + ExecutionResultRow, + ExecutionWithJenkinsInfo, + PotentiallyTimedOutExecution, + RecentExecution, + StaleExecutionSummary, + StuckExecution, + TestRunBasicInfo, + TestRunRow, + TestRunStatusInfo, + TestRunWithUser, +} from './ExecutionRepositoryTypes'; +export abstract class ExecutionRepositoryStatusUtilities extends ExecutionRepositoryLookup { + async countResultsByStatus(executionId: number): Promise<{ passed: number; failed: number; skipped: number; total: number }> { + const rows = await this.testRunResultRepository.query(` + SELECT + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) AS passed, + SUM(CASE WHEN status IN ('failed', 'error') THEN 1 ELSE 0 END) AS failed, + SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped, + COUNT(*) AS total + FROM Auto_TestRunResults + WHERE execution_id = ? + `, [executionId]) as Array<{ passed: string; failed: string; skipped: string; total: string }>; + + if (!rows || rows.length === 0) { + return { passed: 0, failed: 0, skipped: 0, total: 0 }; + } + return { + passed: Number(rows[0].passed ?? 0), + failed: Number(rows[0].failed ?? 0), + skipped: Number(rows[0].skipped ?? 0), + total: Number(rows[0].total ?? 0), + }; + } + + /** + * 通过 executionId(Auto_TestCaseTaskExecutions.id)找到关联的 Auto_TestRun, + * 并同步更新其 status/passedCases/failedCases/skippedCases/durationMs 等统计字段。 + * 用于修复 handleCallback 路径(/api/executions/callback)只更新了 TaskExecution 而未更新 TestRun 的问题。 + */ + + protected mapStatusForTestRun(status: string): 'pending' | 'running' | 'success' | 'failed' | 'aborted' { + // 将 'cancelled' 映射为 'aborted' 以匹配 TestRun 的枚举 + if (status === 'cancelled') { + return 'aborted'; + } + return status as 'pending' | 'running' | 'success' | 'failed' | 'aborted'; + } + + /** + * 归一化回调中的单用例状态,避免非标准值导致写库失败并残留占位 error。 + */ + protected normalizeCaseResultStatus(status: string): TestRunResultStatusType { + const normalized = String(status ?? '').trim().toLowerCase(); + + if (normalized === 'passed' || normalized === 'success' || normalized === 'pass') { + return TestRunResultStatus.PASSED; + } + + if (normalized === 'failed' || normalized === 'fail') { + return TestRunResultStatus.FAILED; + } + + if (normalized === 'skipped' || normalized === 'skip') { + return TestRunResultStatus.SKIPPED; + } + + if (normalized === 'error') { + return TestRunResultStatus.ERROR; + } + + // 未知状态统一归为 error,便于前端识别并排查 + return TestRunResultStatus.ERROR; + } + + /** + * 修复10: completeBatch 拆分 - 多策略解析 executionId + * + * 策略优先级(由高到低): + * 1. 从 Auto_TestRun.execution_id 直接查询(最可靠,run 与 execution 的强绑定) + * 2. 调用方传入的缓存值(仅在 run 尚未绑定 execution_id 时使用) + * 3. 时间窗口 + 触发者反查(兜底,可能不够精确) + */ +} diff --git a/server/repositories/ExecutionRepositoryTypes.ts b/server/repositories/ExecutionRepositoryTypes.ts new file mode 100644 index 0000000..b32034d --- /dev/null +++ b/server/repositories/ExecutionRepositoryTypes.ts @@ -0,0 +1,210 @@ +import { DataSource, QueryRunner, In, Repository, QueryDeepPartialEntity } from 'typeorm'; +import { TaskExecution, TestRun, TestRunResult, TestCase } from '../entities/index'; +import { BaseRepository } from './BaseRepository'; +import { User } from '../entities/User'; +import logger from '../utils/logger'; +import { LOG_CONTEXTS } from '../config/logging'; +import { + TestRunStatus, + TaskExecutionStatus, + TestRunResultStatus, + TestRunResultStatusType, + TestRunTriggerTypeType, +} from '../../shared/types/execution'; + +// ============================================================================ +// 接口定义 +// ============================================================================ + +/** + * 带用户信息的 TaskExecution 接口 + */ +export interface TaskExecutionWithUser extends Omit { + executedByUser?: User; + executedByName?: string; +} + +/** + * 带用户信息的 TestRun 接口 + */ +export interface TestRunWithUser extends Omit { + triggerByUser?: User; + triggerByName?: string; +} + +/** + * 执行详情接口 + */ +export interface ExecutionDetail { + execution: TaskExecutionWithUser; + results: TestRunResult[]; +} + +/** + * 最近运行记录接口 + */ +export interface RecentExecution { + id: number; + taskId?: number; + taskName?: string; + status: string; + totalCases: number; + passedCases: number; + failedCases: number; + skippedCases: number; + duration: number; + executedBy: number; + executedByName?: string; + startTime?: Date; + endTime?: Date; + createdAt?: Date; + updatedAt?: Date; +} + +/** + * 执行结果行接口(原生 SQL 查询结果) + */ +export interface ExecutionResultRow { + id: number; + execution_id: number; + case_id: number; + case_name: string; + module: string; + priority: string; + type: string; + status: string; + start_time: Date | null; + end_time: Date | null; + duration: number | null; + error_message: string | null; + error_stack: string | null; + screenshot_path: string | null; + log_path: string | null; + assertions_total: number | null; + assertions_passed: number | null; + response_data: string | null; + created_at: Date; +} + +/** + * 测试运行行接口(原生 SQL 查询结果) + */ +export interface TestRunRow { + id: number; + project_id: number | null; + project_name: string; + status: string; + trigger_type: string; + trigger_by: number; + trigger_by_name: string; + jenkins_job: string | null; + jenkins_build_id: string | null; + jenkins_url: string | null; + abort_reason: string | null; + total_cases: number; + passed_cases: number; + failed_cases: number; + skipped_cases: number; + duration_ms: number; + start_time: Date | null; + end_time: Date | null; + created_at: Date | null; +} + +/** + * 潜在超时执行接口 + */ +export interface PotentiallyTimedOutExecution { + id: number; + jenkinsJob: string | null; + jenkinsBuildId: string | null; + startTime: Date | null; +} + +/** + * Jenkins 执行信息接口 + */ +export interface ExecutionWithJenkinsInfo { + id: number; + status: string; + jenkinsJob: string | null; + jenkinsBuildId: string | null; +} + +/** + * 卡住的执行接口 + */ +export interface StuckExecution { + id: number; + status: string; + jenkinsJob: string | null; + jenkinsBuildId: string | null; + startTime: Date | null; + durationSeconds: number; +} + +/** + * 历史卡住执行汇总 + */ +export interface StaleExecutionSummary { + stalePendingNoStartCount: number; + staleStartedCount: number; + totalStaleCount: number; + latestStalePendingCreatedAt: Date | null; +} + +/** + * 测试运行基本信息接口 + */ +export interface TestRunBasicInfo { + totalCases: number; +} + +/** + * 测试运行状态信息接口 + */ +export interface TestRunStatusInfo { + id: number; + executionId: number | null; + status: string; + jenkinsJob: string | null; + jenkinsBuildId: string | null; + jenkinsUrl: string | null; + startTime: Date | null; +} + +// ============================================================================ +// completeBatch 内部类型 +// ============================================================================ + +/** completeBatch 方法接收的单条用例结果 */ +export interface BatchCaseResult { + /** caseId 可为空(如 pytest 等不携带 ID 的框架),此时通过 caseName fallback 匹配 */ + caseId?: number; + caseName: string; + status: string; + duration: number; + errorMessage?: string; + stackTrace?: string; + screenshotPath?: string; + logPath?: string; + assertionsTotal?: number; + assertionsPassed?: number; + responseData?: string; + startTime?: string | number; + endTime?: string | number; +} + +/** completeBatch 方法接收的批次结果 */ +export interface BatchResults { + status: 'success' | 'failed' | 'cancelled' | 'aborted'; + passedCases: number; + failedCases: number; + skippedCases: number; + durationMs: number; + results?: BatchCaseResult[]; +} + +/** + * 运行记录 Repository + */ diff --git a/server/routes/jenkins.ts b/server/routes/jenkins.ts index e43c760..69e9ab7 100644 --- a/server/routes/jenkins.ts +++ b/server/routes/jenkins.ts @@ -1,2585 +1,12 @@ -import { Router, Request, Response } from 'express'; -import { In } from 'typeorm'; -import { executionService, type Auto_TestRunResultsInput } from '../services/ExecutionService'; -import { jenkinsService } from '../services/JenkinsService'; -import { jenkinsStatusService } from '../services/JenkinsStatusService'; -import { taskSchedulerService } from '../services/TaskSchedulerService'; -import { callbackQueue, type CallbackPayload } from '../services/CallbackQueue'; -import { ipWhitelistMiddleware, rateLimitMiddleware } from '../middleware/JenkinsAuthMiddleware'; -import { requestValidator } from '../middleware/RequestValidator'; -import { generalAuthRateLimiter } from '../middleware/authRateLimiter'; -import { optionalAuth } from '../middleware/auth'; -import logger from '../utils/logger'; -import { buildJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnostics'; -import { persistJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnosticArtifact'; -import { validateScriptPathsInTestRepo } from '../utils/testRepoScriptPathValidator'; -import { LOG_CONTEXTS, LOG_EVENTS, createTimer } from '../config/logging'; -import { AppDataSource, query, queryOne } from '../config/database'; -import { TestCase } from '../entities/TestCase'; -import { hybridSyncService } from '../services/HybridSyncService'; -import { executionMonitorService } from '../services/ExecutionMonitorService'; -import { - CALLBACK_TERMINAL_STATUSES, - deriveCallbackTerminalStatus, - normalizeCallbackTerminalStatus, -} from '../services/ExecutionService/callbackStatus'; +import { Router } from 'express'; +import { registerJenkinsCallbackToolRoutes } from './jenkinsCallbackToolRoutes'; +import { registerJenkinsDiagnosticRoutes } from './jenkinsDiagnosticRoutes'; +import { registerJenkinsExecutionRoutes } from './jenkinsExecutionRoutes'; const router = Router(); -// ──────────────────────────────────────────────────────────────────────────── -// 常量定义 -// ──────────────────────────────────────────────────────────────────────────── - -/** 回调兜底同步默认延迟(毫秒) */ -const DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS = 45_000; -/** 回调兜底同步最小延迟(毫秒) */ -const MIN_CALLBACK_FALLBACK_SYNC_DELAY_MS = 10_000; -/** Jenkins 健康检查超时(毫秒) */ -const HEALTH_CHECK_TIMEOUT_MS = 5_000; -/** Jenkins 健康检查默认 URL */ -const DEFAULT_JENKINS_URL = 'http://jenkins.wiac.xyz'; -/** Jenkins 健康检查默认用户 */ -const DEFAULT_JENKINS_USER = 'root'; -/** 触发前 Jenkins 预检查默认超时(毫秒) */ -const DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS = 8_000; -/** 触发前 Jenkins 预检查超时(毫秒) */ -const TRIGGER_PRECHECK_TIMEOUT_MS = Math.max( - 1_000, - Number.parseInt( - process.env.JENKINS_TRIGGER_PRECHECK_TIMEOUT_MS ?? String(DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS), - 10 - ) || DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS -); -/** 触发前 Jenkins 预检查重试次数(总尝试次数 = 1 + retries) */ -const TRIGGER_PRECHECK_RETRIES = Math.max( - 0, - Math.min(3, Number.parseInt(process.env.JENKINS_TRIGGER_PRECHECK_RETRIES ?? '1', 10) || 1) -); -/** 触发前 Jenkins 预检查重试间隔(毫秒) */ -const TRIGGER_PRECHECK_RETRY_DELAY_MS = Math.max( - 200, - Number.parseInt(process.env.JENKINS_TRIGGER_PRECHECK_RETRY_DELAY_MS ?? '600', 10) || 600 -); -/** 是否启用触发前 Jenkins 预检查 */ -// 注:Jenkins 预检查默认禁用(当 Jenkins 网络不稳定时) -// 设置 JENKINS_TRIGGER_PRECHECK_ENABLED=true 以启用 -// 启用后,当 Jenkins 无法连接时,任务触发请求会被拒绝 (503 Service Unavailable) -const TRIGGER_PRECHECK_ENABLED = (process.env.JENKINS_TRIGGER_PRECHECK_ENABLED ?? 'false') !== 'false'; - -/** - * [P2-B] 注册 CallbackQueue 消费者 - * 将 completeBatchExecution + releaseSlot 的整个处理流程注入队列 worker - * 在路由模块初始化时立即注册,确保消费者在第一个请求到来之前已就绪 - * - * 支持两种模式: - * 1. 全量回调:Jenkins 解析结果后发送完整数据(兼容旧模式) - * 2. 轻量化回调:Jenkins 仅发送 buildNumber,服务端主动解析结果 - */ -callbackQueue.register(async (payload: CallbackPayload) => { - let shouldReleaseSlot = false; - try { - let finalPayload = payload; - - // ─── 轻量化回调:服务端主动解析结果 ───────────────────────────── - if (payload.needsServerParsing) { - logger.info('[CallbackQueue] Lightweight callback detected, parsing results from Jenkins', { - runId: payload.runId, - buildNumber: payload.buildNumber, - }, LOG_CONTEXTS.JENKINS); - - try { - // 从数据库获取执行记录,提取 jenkinsJob 名称 - const batch = await executionService.getBatchExecution(payload.runId); - const execution = batch?.execution; - const buildNumber = payload.buildNumber ?? execution?.jenkinsBuildId; - - if (execution?.jenkinsJob && buildNumber) { - const testResults = await jenkinsStatusService.parseBuildResults( - execution.jenkinsJob as string, - String(buildNumber) - ); - - if (testResults) { - const reportedStatus = normalizeCallbackTerminalStatus(payload.status); - // 使用解析结果覆盖 payload - finalPayload = { - runId: payload.runId, - status: deriveCallbackTerminalStatus({ - reportedStatus, - passedCases: testResults.passedCases, - failedCases: testResults.failedCases, - skippedCases: testResults.skippedCases, - }), - passedCases: testResults.passedCases, - failedCases: testResults.failedCases, - skippedCases: testResults.skippedCases, - durationMs: testResults.duration || payload.durationMs, - results: testResults.results.map(r => ({ - caseId: r.caseId, - caseName: r.caseName, - status: r.status, - duration: r.duration, - errorMessage: r.errorMessage, - stackTrace: r.stackTrace, - })), - }; - - logger.info('[CallbackQueue] Successfully parsed results from Jenkins', { - runId: payload.runId, - buildNumber, - status: finalPayload.status, - passedCases: finalPayload.passedCases, - failedCases: finalPayload.failedCases, - skippedCases: finalPayload.skippedCases, - }, LOG_CONTEXTS.JENKINS); - } else { - // 解析失败,降级为使用构建状态 - logger.warn('[CallbackQueue] Failed to parse results from Jenkins, falling back to build status', { - runId: payload.runId, - buildNumber, - fallbackStatus: payload.status, - }, LOG_CONTEXTS.JENKINS); - } - } else { - logger.warn('[CallbackQueue] No jenkins job/build number found for execution, cannot parse results', { - runId: payload.runId, - buildNumber, - jenkinsJob: execution?.jenkinsJob, - jenkinsBuildId: execution?.jenkinsBuildId, - }, LOG_CONTEXTS.JENKINS); - } - } catch (parseError) { - logger.error('Failed to parse build results in lightweight callback', { - event: LOG_EVENTS.JENKINS_CALLBACK_PARSE_FAILED, - runId: payload.runId, - buildNumber: payload.buildNumber, - error: parseError instanceof Error ? parseError.message : String(parseError), - }, LOG_CONTEXTS.JENKINS); - // 解析异常,继续使用原始 payload(降级处理) - } - } - - const normalizedReportedStatus = normalizeCallbackTerminalStatus(finalPayload.status); - const hasCallbackSummary = (finalPayload.passedCases + finalPayload.failedCases + finalPayload.skippedCases) > 0; - finalPayload = { - ...finalPayload, - status: hasCallbackSummary - ? deriveCallbackTerminalStatus({ - reportedStatus: normalizedReportedStatus, - passedCases: finalPayload.passedCases, - failedCases: finalPayload.failedCases, - skippedCases: finalPayload.skippedCases, - }) - : normalizedReportedStatus, - }; - - await executionService.completeBatchExecution(finalPayload.runId, { - status: finalPayload.status, - passedCases: finalPayload.passedCases, - failedCases: finalPayload.failedCases, - skippedCases: finalPayload.skippedCases, - durationMs: finalPayload.durationMs, - results: finalPayload.results as Parameters[1]['results'], - }); - // 只在成功完成后标记需要释放槽位 - shouldReleaseSlot = true; - } catch (error) { - // 如果 completeBatchExecution 失败,让 CallbackQueue 的重试机制处理 - // 不释放槽位,避免重复释放 - logger.warn('[CallbackQueue] completeBatchExecution failed, will retry', { - runId: payload.runId, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.EXECUTION); - throw error; // 重新抛出错误以触发重试 - } finally { - // 只在成功完成后释放并发槽位 - if (shouldReleaseSlot) { - taskSchedulerService.releaseSlotByRunId(payload.runId); - logger.debug('[CallbackQueue] Slot released after successful completion', { - runId: payload.runId, - }, LOG_CONTEXTS.EXECUTION); - } - } -}); - -const CALLBACK_FALLBACK_SYNC_DELAY_MS = Math.max( - MIN_CALLBACK_FALLBACK_SYNC_DELAY_MS, - Number.parseInt( - process.env.CALLBACK_FALLBACK_SYNC_DELAY_MS ?? String(DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS), - 10 - ) || DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS -); - -/** - * 当 Jenkins 回调丢失时,延迟触发一次兜底同步,避免状态长时间停留在 running/pending。 - */ -function scheduleCallbackFallbackSync(runId: number, source: 'run-case' | 'run-batch'): void { - const timer = setTimeout(async () => { - try { - const detail = await executionService.getTestRunDetailRow(runId); - const currentStatus = String(detail.status ?? ''); - - if (!['pending', 'running'].includes(currentStatus)) { - logger.debug('[callback-fallback] execution already finalized, skipping sync', { - runId, - source, - currentStatus, - }, LOG_CONTEXTS.JENKINS); - return; - } - - const syncResult = await executionService.syncExecutionStatusFromJenkins(runId); - logger.info('[callback-fallback] fallback sync executed', { - runId, - source, - currentStatus, - delayMs: CALLBACK_FALLBACK_SYNC_DELAY_MS, - syncSuccess: syncResult.success, - syncUpdated: syncResult.updated, - jenkinsStatus: syncResult.jenkinsStatus, - message: syncResult.message, - }, LOG_CONTEXTS.JENKINS); - } catch (error) { - logger.warn('[callback-fallback] fallback sync failed', { - runId, - source, - delayMs: CALLBACK_FALLBACK_SYNC_DELAY_MS, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.JENKINS); - } - }, CALLBACK_FALLBACK_SYNC_DELAY_MS); - - timer.unref?.(); -} - -/** - * 构造 Jenkins 回调 URL:兼容配置基础地址或完整 callback 路径。 - */ -function buildCallbackUrl(): string { - const configuredBase = (process.env.API_CALLBACK_URL ?? 'http://localhost:3000').trim(); - - // 兼容老配置:如果配置已包含 callback 路径,优先直接使用。 - const trimmed = configuredBase.replace(/\/+$/, ''); - if (trimmed.endsWith('/api/jenkins/callback')) { - warnIfCallbackUrlIsLocal(trimmed); - return trimmed; - } - - const callbackUrl = `${trimmed}/api/jenkins/callback`; - warnIfCallbackUrlIsLocal(callbackUrl); - return callbackUrl; -} - -function warnIfCallbackUrlIsLocal(callbackUrl: string): void { - try { - const callbackHost = new URL(callbackUrl).hostname.toLowerCase(); - const jenkinsHost = new URL(process.env.JENKINS_URL || DEFAULT_JENKINS_URL).hostname.toLowerCase(); - const localHosts = new Set(['localhost', '127.0.0.1', '::1']); - - if (localHosts.has(callbackHost) && !localHosts.has(jenkinsHost)) { - logger.warn('Jenkins callback URL points to localhost while Jenkins is remote', { - event: 'JENKINS_CALLBACK_URL_LOCALHOST_FOR_REMOTE', - callbackUrl, - jenkinsHost, - suggestion: 'Set API_CALLBACK_URL to a URL that Jenkins can reach, otherwise callback may fail with 403 or never reach this service.', - }, LOG_CONTEXTS.JENKINS); - } - } catch (error) { - logger.warn('Failed to validate Jenkins callback URL', { - event: 'JENKINS_CALLBACK_URL_VALIDATE_FAILED', - callbackUrl, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.JENKINS); - } -} - -async function recordTriggerFailure( - runId: number, - caseIds: number[], - scriptPaths: string[], - callbackUrl: string, - source: 'run-case' | 'run-batch', - triggerResult: { message: string; errorCategory: 'none' | 'network' | 'auth_failed' | 'not_found' | 'bad_request' | 'rate_limited' | 'server_error' } -): Promise { - const config = jenkinsService.getConfigInfo(); - const persisted = await persistJenkinsTriggerFailureDiagnostic(triggerResult, { - runId, - source, - baseUrl: config?.baseUrl, - jobName: config?.jobs.api, - callbackUrl, - caseIds, - scriptPaths, - }).catch(async (error: unknown) => { - logger.warn('Failed to persist Jenkins trigger diagnostic artifact', { - runId, - error: error instanceof Error ? error.message : String(error), - }, LOG_CONTEXTS.JENKINS); - - return { - publicPath: undefined, - diagnostic: buildJenkinsTriggerFailureDiagnostic(triggerResult, { - baseUrl: config?.baseUrl, - jobName: config?.jobs.api, - callbackUrl, - caseIds, - scriptPaths, - }), - }; - }); - - await executionService.recordTriggerFailureDiagnostics({ - runId, - caseIds, - errorMessage: persisted.diagnostic.errorMessage, - errorStack: persisted.diagnostic.errorStack, - logPath: persisted.publicPath, - }); - await executionService.markExecutionAborted(runId, persisted.diagnostic.abortReason); -} - -/** - * 执行触发前的 Jenkins 连通性预检查。 - * 目标:在 Jenkins 不可用时快速失败,避免创建后立刻变成 aborted 的运行记录。 - */ -async function runJenkinsTriggerPrecheck(source: 'run-case' | 'run-batch'): Promise<{ ok: true } | { ok: false; reason: string }> { - const configError = jenkinsService.getTriggerConfigurationError(); - if (configError) { - logger.warn('[trigger-precheck] Jenkins trigger configuration invalid', { - source, - reason: configError, - }, LOG_CONTEXTS.JENKINS); - return { ok: false, reason: configError }; - } - - if (!TRIGGER_PRECHECK_ENABLED) { - return { ok: true }; - } - - const maxAttempts = 1 + TRIGGER_PRECHECK_RETRIES; - const reasons: string[] = []; - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - let timeoutId: NodeJS.Timeout | undefined; - - try { - const timeoutPromise = new Promise<{ connected: false; message: string }>((resolve) => { - timeoutId = setTimeout(() => { - resolve({ - connected: false, - message: `Jenkins precheck timeout after ${TRIGGER_PRECHECK_TIMEOUT_MS}ms`, - }); - }, TRIGGER_PRECHECK_TIMEOUT_MS); - timeoutId.unref?.(); - }); - - const checkResult = await Promise.race([ - jenkinsService.testConnection(), - timeoutPromise, - ]); - - if (checkResult.connected) { - if (attempt > 1) { - logger.info('[trigger-precheck] Jenkins precheck recovered after retry', { - source, - attempt, - maxAttempts, - }, LOG_CONTEXTS.JENKINS); - } - return { ok: true }; - } - - const reason = checkResult.message || 'Jenkins unavailable'; - reasons.push(reason); - - // 配置缺失属于确定性失败,无需重试 - if (reason.includes('not configured')) { - break; - } - - logger.warn('[trigger-precheck] Jenkins unavailable in attempt', { - source, - attempt, - maxAttempts, - reason, - timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, - }, LOG_CONTEXTS.JENKINS); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - reasons.push(reason); - logger.warn('[trigger-precheck] Jenkins precheck attempt failed with exception', { - source, - attempt, - maxAttempts, - reason, - timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, - }, LOG_CONTEXTS.JENKINS); - } finally { - if (timeoutId) clearTimeout(timeoutId); - } - - if (attempt < maxAttempts) { - await new Promise((resolve) => { - const timer = setTimeout(() => resolve(), TRIGGER_PRECHECK_RETRY_DELAY_MS); - timer.unref?.(); - }); - } - } - - const reason = reasons[reasons.length - 1] || 'Jenkins unavailable'; - logger.warn('[trigger-precheck] Jenkins unavailable, rejecting trigger request after retries', { - source, - maxAttempts, - reason, - reasons, - timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, - retryDelayMs: TRIGGER_PRECHECK_RETRY_DELAY_MS, - }, LOG_CONTEXTS.JENKINS); - - return { ok: false, reason }; -} - -/** - * 解析并去重脚本路径 - */ -async function resolveScriptPaths(caseIds: number[]): Promise<{ scriptPaths: string[]; missingCaseIds: number[] }> { - if (!Array.isArray(caseIds) || caseIds.length === 0) { - return { scriptPaths: [], missingCaseIds: [] }; - } - - const cases = await AppDataSource.getRepository(TestCase).find({ - where: { - id: In(caseIds), - enabled: true, - }, - select: ['id', 'scriptPath'], - }); - - const scriptPathCaseIds = new Set(); - const normalizedPaths = new Set(); - - for (const item of cases) { - const path = item.scriptPath?.trim(); - if (path) { - scriptPathCaseIds.add(item.id); - normalizedPaths.add(path); - } - } - - const missingCaseIds = caseIds.filter(id => !scriptPathCaseIds.has(id)); - - return { - scriptPaths: Array.from(normalizedPaths), - missingCaseIds, - }; -} - -async function preflightExecutableScriptPaths(caseIds: number[]): Promise< - | { ok: true; scriptPaths: string[] } - | { - ok: false; - statusCode: number; - message: string; - details: { - reason: 'missing_script_path' | 'script_path_not_found_in_repo'; - caseIds?: number[]; - missingPaths?: string[]; - }; - } -> { - const { scriptPaths, missingCaseIds } = await resolveScriptPaths(caseIds); - - if (missingCaseIds.length > 0) { - return { - ok: false, - statusCode: 400, - message: missingCaseIds.length === 1 - ? `测试用例 ${missingCaseIds[0]} 未配置 script_path,请先同步或修正后再执行` - : `存在未配置 script_path 的测试用例,请先同步或修正后再执行:${missingCaseIds.join(', ')}`, - details: { - reason: 'missing_script_path', - caseIds: missingCaseIds, - }, - }; - } - - const testRepoConfig = jenkinsService.getTestRepoConfig(); - const missingPaths = testRepoConfig - ? (await validateScriptPathsInTestRepo({ - repoUrl: testRepoConfig.repoUrl, - branch: testRepoConfig.branch, - scriptPaths, - })).missingPaths - : []; - - if (missingPaths.length > 0) { - return { - ok: false, - statusCode: 400, - message: missingPaths.length === 1 - ? `测试仓库中不存在脚本路径:${missingPaths[0]}` - : `测试仓库中存在无效脚本路径,请先同步或修正后再执行:${missingPaths.join(', ')}`, - details: { - reason: 'script_path_not_found_in_repo', - missingPaths, - }, - }; - } - - return { ok: true, scriptPaths }; -} - -/** - * 净化错误消息,移除敏感信息以防止信息泄露 - * @param error 原始错误对象 - * @param context 错误上下文,用于日志记录 - * @returns 净化后的错误消息 - */ -function sanitizeErrorMessage(error: unknown, context: string): string { - const originalMessage = error instanceof Error ? error.message : 'Unknown error'; - - // 记录详细错误信息到服务器日志 - logger.error(`${context} - Detailed error info`, { - event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, - message: originalMessage, - stack: error instanceof Error ? error.stack : undefined, - timestamp: new Date().toISOString(), - context, - }, LOG_CONTEXTS.JENKINS); - - // 检查是否包含敏感信息关键词 - const sensitiveKeywords = [ - 'password', 'token', 'secret', 'key', 'credential', - ]; - - const lowerMessage = originalMessage.toLowerCase(); - const containsSensitiveInfo = sensitiveKeywords.some(keyword => - lowerMessage.includes(keyword.toLowerCase()) - ); - - if (containsSensitiveInfo) { - // 包含敏感信息时返回通用错误消息 - return 'An internal error occurred. Please contact support if the issue persists.'; - } - - // 生产环境返回简化但有意义的错误消息(移除路径、IP 等敏感信息) - if (process.env.NODE_ENV === 'production') { - return originalMessage - .replace(/\/[^\s]+/g, '[path]') // 替换文件路径 - .replace(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/g, '[ip]') // 替换 IP 地址 - .replace(/:\d+/g, ':[port]') // 替换端口号 - .replace(/localhost/gi, '[host]') // 替换 localhost - .replace(/127\.0\.0\.1/g, '[host]'); // 替换本地 IP - } - - // 开发环境返回原始消息 - return originalMessage; -} - -function resolveExecutionBusinessError(error: unknown): { - statusCode: number; - message: string; - details: { reason: 'inactive_case' | 'inactive_cases'; caseIds: number[] }; -} | null { - const originalMessage = error instanceof Error ? error.message : String(error ?? ''); - const noActiveCasesMatch = originalMessage.match(/No active test cases found with IDs:\s*(.+)$/i); - if (!noActiveCasesMatch) return null; - - const caseIds = noActiveCasesMatch[1] - .split(',') - .map(part => Number.parseInt(part.trim(), 10)) - .filter(Number.isFinite); - - if (caseIds.length === 0) return null; - - if (caseIds.length === 1) { - return { - statusCode: 400, - message: `测试用例 ${caseIds[0]} 未启用,请先启用后再执行`, - details: { - reason: 'inactive_case', - caseIds, - }, - }; - } - - return { - statusCode: 400, - message: `存在未启用的测试用例,请先启用后再执行:${caseIds.join(', ')}`, - details: { - reason: 'inactive_cases', - caseIds, - }, - }; -} - -/** - * 规范化 Jenkins 回调中的 results 载荷,兼容 camelCase/snake_case 字段。 - * 目标:确保后续 completeBatchExecution 能稳定回写用例明细,避免残留占位 error。 - */ -function normalizeCallbackResults(results: unknown[]): Auto_TestRunResultsInput[] { - const toNumber = (value: unknown): number | undefined => { - if (typeof value === 'number' && Number.isFinite(value)) return value; - if (typeof value === 'string' && value.trim() !== '') { - const parsed = Number(value); - if (Number.isFinite(parsed)) return parsed; - } - return undefined; - }; - - const toOptionalString = (value: unknown): string | undefined => { - if (typeof value === 'string') { - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; - } - return undefined; - }; - - const normalizeStatus = (value: unknown): Auto_TestRunResultsInput['status'] => { - const rawStatus = String(value ?? '').trim().toLowerCase(); - if (rawStatus === 'passed' || rawStatus === 'success' || rawStatus === 'pass') return 'passed'; - if (rawStatus === 'failed' || rawStatus === 'fail') return 'failed'; - if (rawStatus === 'skipped' || rawStatus === 'skip') return 'skipped'; - - // 记录未知状态 - logger.warn('Unknown test result status, treating as error', { - rawStatus, - originalValue: value, - }, LOG_CONTEXTS.JENKINS); - return 'error'; - }; - - return results.flatMap((item): Auto_TestRunResultsInput[] => { - if (!item || typeof item !== 'object' || Array.isArray(item)) return []; - const row = item as Record; - - const caseIdRaw = toNumber(row['caseId'] ?? row['case_id']); - const caseName = toOptionalString(row['caseName'] ?? row['case_name']); - - // 允许 caseId=0 的结果通过(如 pytest 等框架不携带 caseId 的场景), - // 由 updateTestResult 的 caseName fallback 机制完成匹配。 - // 但若 caseId 和 caseName 同时缺失,则过滤掉(无法匹配任何占位符记录)。 - const hasValidCaseId = caseIdRaw && caseIdRaw > 0; - if (!hasValidCaseId && !caseName) { - logger.warn('Filtered out test result: missing both caseId and caseName', { - row, - caseId: caseIdRaw, - caseName, - }, LOG_CONTEXTS.JENKINS); - return []; - } - - if (!hasValidCaseId) { - logger.debug('Test result has no valid caseId, will use caseName fallback matching', { - caseId: caseIdRaw, - caseName, - }, LOG_CONTEXTS.JENKINS); - } - - const durationRaw = toNumber(row['duration'] ?? row['durationMs'] ?? row['duration_ms']); - const assertionsTotal = toNumber(row['assertionsTotal'] ?? row['assertions_total']); - const assertionsPassed = toNumber(row['assertionsPassed'] ?? row['assertions_passed']); - const startTime = row['startTime'] ?? row['start_time']; - const endTime = row['endTime'] ?? row['end_time']; - const responseDataRaw = row['responseData'] ?? row['response_data']; - - return [{ - caseId: caseIdRaw, - caseName: caseName || `case_${caseIdRaw}`, - status: normalizeStatus(row['status']), - duration: durationRaw !== undefined ? Math.max(0, durationRaw) : 0, - errorMessage: toOptionalString(row['errorMessage'] ?? row['error_message']), - stackTrace: toOptionalString(row['stackTrace'] ?? row['errorStack'] ?? row['error_stack']), - screenshotPath: toOptionalString(row['screenshotPath'] ?? row['screenshot_path']), - logPath: toOptionalString(row['logPath'] ?? row['log_path']), - assertionsTotal, - assertionsPassed, - responseData: typeof responseDataRaw === 'string' - ? responseDataRaw - : (responseDataRaw !== undefined ? JSON.stringify(responseDataRaw) : undefined), - startTime: typeof startTime === 'string' || typeof startTime === 'number' ? startTime : undefined, - endTime: typeof endTime === 'string' || typeof endTime === 'number' ? endTime : undefined, - }]; - }); -} - -/** - * POST /api/jenkins/trigger - * 触发 Jenkins Job 执行 - * - * 此接口创建运行记录并返回 executionId,供 Jenkins 后续回调使用 - * 支持两种模式: - * 1. 直接传入 caseIds 数组 - * 2. 传入 taskId,自动从数据库查找任务的 caseIds 和任务名称 - */ -router.post('/trigger', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - try { - const triggerBody = (req.body ?? {}) as Record; - let caseIds = triggerBody['caseIds']; - const projectId = typeof triggerBody['projectId'] === 'number' ? triggerBody['projectId'] : 1; - // 优先使用认证用户 ID,回退到请求体中的 triggeredBy,最后才用默认值 1(系统管理员) - const triggeredBy = req.user?.id ?? (typeof triggerBody['triggeredBy'] === 'number' ? triggerBody['triggeredBy'] : 1); - const jenkinsJobName = typeof triggerBody['jenkinsJobName'] === 'string' ? triggerBody['jenkinsJobName'] : undefined; - const taskId = typeof triggerBody['taskId'] === 'number' ? triggerBody['taskId'] : undefined; - let taskName: string | undefined; - - // 如果传入了 taskId,从数据库查找任务信息 - if (taskId !== undefined) { - const task = await queryOne<{ id: number; name: string; case_ids: string; project_id: number }>( - 'SELECT id, name, case_ids, project_id FROM Auto_TestCaseTasks WHERE id = ?', - [taskId] - ); - - if (!task) { - return res.status(404).json({ - success: false, - message: `Task with id ${taskId} not found` - }); - } - - taskName = task.name; - - // 如果没有直接传入 caseIds,从任务中解析 - if (!caseIds || !Array.isArray(caseIds) || caseIds.length === 0) { - try { - const parsedCaseIds = JSON.parse(task.case_ids); - if (!Array.isArray(parsedCaseIds) || parsedCaseIds.length === 0) { - logger.warn('Task has empty or invalid case_ids', { - taskId, - case_ids: task.case_ids, - }, LOG_CONTEXTS.JENKINS); - return res.status(400).json({ - success: false, - message: `Task ${taskId} has no valid case_ids configured` - }); - } - caseIds = parsedCaseIds as number[]; - } catch (err) { - logger.error('Failed to parse task case_ids', { - event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, - taskId, - case_ids: task.case_ids, - error: err instanceof Error ? err.message : String(err), - }, LOG_CONTEXTS.JENKINS); - return res.status(500).json({ - success: false, - message: 'Failed to parse task configuration. Invalid JSON format in case_ids field.' - }); - } - } - } - - if (!caseIds || !Array.isArray(caseIds) || caseIds.length === 0) { - return res.status(400).json({ - success: false, - message: 'caseIds is required and must be a non-empty array (or provide a valid taskId with case_ids)' - }); - } - - // 创建运行记录 - const execution = await executionService.triggerTestExecution({ - caseIds: caseIds as number[], - projectId, - triggeredBy, - triggerType: 'jenkins', - jenkinsJob: jenkinsJobName, - taskId, - taskName, - }); - - res.json({ - success: true, - data: { - runId: execution.runId, - totalCases: execution.totalCases, - status: 'pending', - jenkinsJobName: jenkinsJobName || null, - message: 'Execution created. Waiting for Jenkins to start.' - } - }); - } catch (error: unknown) { - const businessError = resolveExecutionBusinessError(error); - if (businessError) { - return res.status(businessError.statusCode).json({ - success: false, - message: businessError.message, - details: businessError.details, - }); - } - - const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_TRIGGER'); - res.status(500).json({ success: false, message: sanitizedMessage }); - } -}); - -/** - * POST /api/jenkins/run-case - * 触发单个用例执行 - * - * 异步队列模式: - * 1. 立即创建执行记录(status=pending) - * 2. 立即返回 runId 给前端(不阻塞) - * 3. 后台通过 enqueueDirectJob 等待并发槽位,槽位可用后再触发 Jenkins - */ -router.post('/run-case', [ - generalAuthRateLimiter, - optionalAuth, - rateLimitMiddleware.limit, - requestValidator.validateSingleExecution -], async (req: Request, res: Response) => { - const timer = createTimer(); - const { caseId, projectId } = req.body; - const triggeredBy: number = req.user?.id ?? (typeof req.body.triggeredBy === 'number' ? req.body.triggeredBy : 1); - const slotLabel = `case:${caseId}`; - - try { - logger.info('Starting single case execution (async queue mode)', { - caseId, - projectId, - triggeredBy, - }, LOG_CONTEXTS.JENKINS); - - const precheck = await runJenkinsTriggerPrecheck('run-case'); - if (!precheck.ok) { - return res.status(503).json({ - success: false, - message: `Jenkins 当前不可用,请稍后重试(${precheck.reason})`, - details: { - reason: precheck.reason, - source: 'run-case-precheck', - retryable: true, - }, - }); - } - - // ── Step 1: 立即创建执行记录(状态 pending)────────────── - const scriptPathPreflight = await preflightExecutableScriptPaths([caseId]); - if (!scriptPathPreflight.ok) { - return res.status(scriptPathPreflight.statusCode).json({ - success: false, - message: scriptPathPreflight.message, - details: scriptPathPreflight.details, - }); - } - - const preflightScriptPaths = scriptPathPreflight.scriptPaths; - - const execution = await executionService.triggerTestExecution({ - caseIds: [caseId], - projectId, - triggeredBy, - triggerType: 'manual', - }); - - logger.info('Execution record created, returning runId immediately', { - runId: execution.runId, - executionId: execution.executionId, - }, LOG_CONTEXTS.JENKINS); - - // ── Step 2: 立即返回 runId,不等待 Jenkins ─────────────── - const duration = timer(); - res.json({ - success: true, - data: { - runId: execution.runId, - status: 'queued', - }, - message: '任务已加入执行队列', - _concurrency: { - slotsUsed: taskSchedulerService.getStatus().running.length, - slotsLimit: taskSchedulerService.getStatus().concurrencyLimit, - directQueued: taskSchedulerService.getStatus().directQueueDepth, - }, - }); - - // ── Step 3: 后台异步等待槽位 + 触发 Jenkins ────────────── - const capturedRunId = execution.runId; - - try { - taskSchedulerService.enqueueDirectJob(slotLabel, async (placeholderRunId: number) => { - // 槽位获取后,用真实 runId 替换占位槽位 - taskSchedulerService.registerDirectSlot(capturedRunId, slotLabel, placeholderRunId); - - try { - // 解析脚本路径 - const callbackUrl = buildCallbackUrl(); - - // 触发 Jenkins - const triggerResult = await jenkinsService.triggerBatchJob( - capturedRunId, - [caseId], - preflightScriptPaths, - callbackUrl, - async (buildNumber: number, buildUrl: string, queueWaitMs: number) => { - const buildId = String(buildNumber); - logger.debug('[dev-10] Build resolved via queueId poll, updating Jenkins info', { - runId: capturedRunId, - buildId, - buildUrl, - queueWaitMs, - }, LOG_CONTEXTS.JENKINS); - await executionService.updateBatchJenkinsInfo(capturedRunId, { buildId, buildUrl }); - scheduleCallbackFallbackSync(capturedRunId, 'run-case'); - }, - async (reason: 'cancelled' | 'timeout') => { - logger.warn('[dev-11] Jenkins queue cancelled/timeout, marking execution as aborted', { - runId: capturedRunId, - reason, - }, LOG_CONTEXTS.JENKINS); - try { - await executionService.markExecutionAborted(capturedRunId, `Jenkins build ${reason}`); - } catch (err) { - logger.warn('[dev-11] Failed to mark execution as aborted', { - runId: capturedRunId, - error: err instanceof Error ? err.message : String(err), - }, LOG_CONTEXTS.JENKINS); - } - taskSchedulerService.releaseSlotByRunId(capturedRunId); - } - ); - - if (!triggerResult.success) { - // Jenkins 触发失败,立即释放槽位 - taskSchedulerService.releaseSlotByRunId(capturedRunId); - // 将执行状态标记为失败 - try { - await recordTriggerFailure(capturedRunId, [caseId], preflightScriptPaths, callbackUrl, 'run-case', triggerResult); - } catch { /* ignore */ } - logger.warn('[run-case] Jenkins trigger failed (async), slot released', { - runId: capturedRunId, - message: triggerResult.message, - }, LOG_CONTEXTS.JENKINS); - } else { - logger.info('[run-case] Jenkins trigger success (async)', { - runId: capturedRunId, - queueId: triggerResult.queueId, - }, LOG_CONTEXTS.JENKINS); - } - } catch (jenkinsErr) { - // Jenkins 执行异常,释放槽位并标记失败 - taskSchedulerService.releaseSlotByRunId(capturedRunId); - try { - await executionService.markExecutionAborted(capturedRunId, `Jenkins error: ${jenkinsErr instanceof Error ? jenkinsErr.message : String(jenkinsErr)}`); - } catch { /* ignore */ } - logger.errorLog(jenkinsErr, '[run-case] Async Jenkins trigger error', { runId: capturedRunId, caseId }); - } - }); - } catch (queueErr) { - // 仅当队列已满才会到这里(enqueueDirectJob 同步抛出) - // runId 已返回给前端,将执行状态标记为失败 - const queueErrMsg = queueErr instanceof Error ? queueErr.message : '并发队列已满'; - logger.warn('[run-case] Queue full, marking execution as aborted', { - runId: capturedRunId, - message: queueErrMsg, - }, LOG_CONTEXTS.JENKINS); - try { - await executionService.markExecutionAborted(capturedRunId, queueErrMsg); - } catch { /* ignore */ } - } - - } catch (error: unknown) { - const duration = timer(); - logger.errorLog(error, 'Single case execution failed (creating record)', { - caseId, - projectId, - durationMs: duration, - }); - - const businessError = resolveExecutionBusinessError(error); - if (businessError) { - return res.status(businessError.statusCode).json({ - success: false, - message: businessError.message, - details: businessError.details, - }); - } - - const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_RUN_CASE'); - res.status(500).json({ success: false, message: sanitizedMessage }); - } -}); - -/** - * POST /api/jenkins/run-batch - * 触发批量用例执行 - * - * 异步队列模式: - * 1. 立即创建执行记录(status=pending) - * 2. 立即返回 runId 给前端(不阻塞) - * 3. 后台通过 enqueueDirectJob 等待并发槽位,槽位可用后再触发 Jenkins - */ -router.post('/run-batch', [ - generalAuthRateLimiter, - optionalAuth, - rateLimitMiddleware.limit, - requestValidator.validateBatchExecution -], async (req: Request, res: Response) => { - const timer = createTimer(); - const { caseIds, projectId } = req.body; - const triggeredBy: number = req.user?.id ?? (typeof req.body.triggeredBy === 'number' ? req.body.triggeredBy : 1); - // label 展示前几个 caseId,避免过长 - const labelIds = (caseIds as number[]).slice(0, 3).join(',') + (caseIds.length > 3 ? `…(${caseIds.length})` : ''); - const slotLabel = `batch:${labelIds}`; - - try { - logger.info('Starting batch case execution (async queue mode)', { - caseCount: caseIds.length, - caseIds, - projectId, - triggeredBy, - }, LOG_CONTEXTS.JENKINS); - - const precheck = await runJenkinsTriggerPrecheck('run-batch'); - if (!precheck.ok) { - return res.status(503).json({ - success: false, - message: `Jenkins 当前不可用,请稍后重试(${precheck.reason})`, - details: { - reason: precheck.reason, - source: 'run-batch-precheck', - retryable: true, - }, - }); - } - - // ── Step 1: 立即创建执行记录(状态 pending)────────────── - const scriptPathPreflight = await preflightExecutableScriptPaths(caseIds); - if (!scriptPathPreflight.ok) { - return res.status(scriptPathPreflight.statusCode).json({ - success: false, - message: scriptPathPreflight.message, - details: scriptPathPreflight.details, - }); - } - - const preflightScriptPaths = scriptPathPreflight.scriptPaths; - - const execution = await executionService.triggerTestExecution({ - caseIds, - projectId, - triggeredBy, - triggerType: 'manual', - }); - - logger.info('Batch execution record created, returning runId immediately', { - runId: execution.runId, - executionId: execution.executionId, - totalCases: execution.totalCases, - }, LOG_CONTEXTS.JENKINS); - - // ── Step 2: 立即返回 runId,不等待 Jenkins ─────────────── - const duration = timer(); - res.json({ - success: true, - data: { - runId: execution.runId, - totalCases: execution.totalCases, - status: 'queued', - }, - message: '任务已加入执行队列', - _concurrency: { - slotsUsed: taskSchedulerService.getStatus().running.length, - slotsLimit: taskSchedulerService.getStatus().concurrencyLimit, - directQueued: taskSchedulerService.getStatus().directQueueDepth, - }, - }); - - // ── Step 3: 后台异步等待槽位 + 触发 Jenkins ────────────── - const capturedRunId = execution.runId; - - try { - taskSchedulerService.enqueueDirectJob(slotLabel, async (placeholderRunId: number) => { - // 槽位获取后,用真实 runId 替换占位槽位 - taskSchedulerService.registerDirectSlot(capturedRunId, slotLabel, placeholderRunId); - - try { - // 解析脚本路径 - const callbackUrl = buildCallbackUrl(); - - // 触发 Jenkins - const triggerResult = await jenkinsService.triggerBatchJob( - capturedRunId, - caseIds, - preflightScriptPaths, - callbackUrl, - async (buildNumber: number, buildUrl: string, queueWaitMs: number) => { - const buildId = String(buildNumber); - logger.debug('[dev-10] Build resolved via queueId poll, updating batch Jenkins info', { - runId: capturedRunId, - buildId, - buildUrl, - queueWaitMs, - }, LOG_CONTEXTS.JENKINS); - await executionService.updateBatchJenkinsInfo(capturedRunId, { buildId, buildUrl }); - scheduleCallbackFallbackSync(capturedRunId, 'run-batch'); - }, - async (reason: 'cancelled' | 'timeout') => { - logger.warn('[dev-11] Batch Jenkins queue cancelled/timeout, marking execution as aborted', { - runId: capturedRunId, - reason, - }, LOG_CONTEXTS.JENKINS); - try { - await executionService.markExecutionAborted(capturedRunId, `Jenkins build ${reason}`); - } catch (err) { - logger.warn('[dev-11] Failed to mark batch execution as aborted', { - runId: capturedRunId, - error: err instanceof Error ? err.message : String(err), - }, LOG_CONTEXTS.JENKINS); - } - taskSchedulerService.releaseSlotByRunId(capturedRunId); - } - ); - - if (!triggerResult.success) { - taskSchedulerService.releaseSlotByRunId(capturedRunId); - try { - await recordTriggerFailure(capturedRunId, caseIds, preflightScriptPaths, callbackUrl, 'run-batch', triggerResult); - } catch { /* ignore */ } - logger.warn('[run-batch] Jenkins trigger failed (async), slot released', { - runId: capturedRunId, - message: triggerResult.message, - }, LOG_CONTEXTS.JENKINS); - } else { - logger.info('[run-batch] Jenkins trigger success (async)', { - runId: capturedRunId, - queueId: triggerResult.queueId, - }, LOG_CONTEXTS.JENKINS); - } - } catch (jenkinsErr) { - taskSchedulerService.releaseSlotByRunId(capturedRunId); - try { - await executionService.markExecutionAborted(capturedRunId, `Jenkins error: ${jenkinsErr instanceof Error ? jenkinsErr.message : String(jenkinsErr)}`); - } catch { /* ignore */ } - logger.errorLog(jenkinsErr, '[run-batch] Async Jenkins trigger error', { runId: capturedRunId, caseIds }); - } - }); - } catch (queueErr) { - const queueErrMsg = queueErr instanceof Error ? queueErr.message : '并发队列已满'; - logger.warn('[run-batch] Queue full, marking execution as aborted', { - runId: capturedRunId, - message: queueErrMsg, - }, LOG_CONTEXTS.JENKINS); - try { - await executionService.markExecutionAborted(capturedRunId, queueErrMsg); - } catch { /* ignore */ } - } - - } catch (error: unknown) { - const duration = timer(); - logger.errorLog(error, 'Batch case execution failed (creating record)', { - caseIds, - projectId, - durationMs: duration, - }); - - const businessError = resolveExecutionBusinessError(error); - if (businessError) { - return res.status(businessError.statusCode).json({ - success: false, - message: businessError.message, - details: businessError.details, - }); - } - - const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_RUN_BATCH'); - res.status(500).json({ success: false, message: sanitizedMessage }); - } -}); - -/** - * GET /api/jenkins/tasks/:taskId/cases - * 获取任务关联的用例列表 - * - * Jenkins Job 可以调用此接口获取需要执行的用例信息 - */ -router.get('/tasks/:taskId/cases', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - try { - const taskId = parseInt(req.params.taskId); - if (isNaN(taskId) || taskId <= 0) { - return res.status(400).json({ - success: false, - message: 'Invalid taskId parameter. Must be a positive integer.' - }); - } - const cases = await executionService.getRunCases(taskId); - - res.json({ - success: true, - data: cases - }); - } catch (error: unknown) { - logger.errorLog(error, 'Failed to get task cases', { - event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, - taskId: req.params.taskId, - }); - const message = error instanceof Error ? error.message : 'Unknown error'; - res.status(500).json({ success: false, message }); - } -}); - -/** - * GET /api/jenkins/status/:executionId - * 查询执行状态(预留接口) - * - * 用于查询 Jenkins Job 的执行状态 - */ -router.get('/status/:executionId', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - try { - const executionId = parseInt(req.params.executionId); - if (isNaN(executionId) || executionId <= 0) { - return res.status(400).json({ - success: false, - message: 'Invalid executionId parameter. Must be a positive integer.' - }); - } - const detail = await executionService.getExecutionDetail(executionId); - - if (!detail || !detail.execution) { - return res.status(404).json({ success: false, message: 'Execution not found' }); - } - - const execution = detail.execution as unknown as Record; - - res.json({ - success: true, - data: { - executionId, - status: execution['status'], - totalCases: execution['total_cases'], - passedCases: execution['passed_cases'], - failedCases: execution['failed_cases'], - skippedCases: execution['skipped_cases'], - startTime: execution['start_time'], - endTime: execution['end_time'], - duration: execution['duration'], - // Jenkins 相关字段(预留) - jenkinsStatus: null, - buildNumber: null, - consoleUrl: null - } - }); - } catch (error: unknown) { - logger.errorLog(error, 'Failed to get execution status', { - event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, - executionId: req.params.executionId, - }); - const message = error instanceof Error ? error.message : 'Unknown error'; - res.status(500).json({ success: false, message }); - } -}); - -/** - * POST /api/jenkins/callback - * Jenkins 执行结果回调接口 - * 通过 IP 白名单验证,无需额外认证 - * 注意:此接口不使用 generalAuthRateLimiter,避免高并发回调时触发 429 - * 安全由 ipWhitelistMiddleware 白名单保护,并使用专用的 rateLimitMiddleware - */ -router.post('/callback', [ - ipWhitelistMiddleware.verify, - rateLimitMiddleware.limit, - requestValidator.validateCallback -], (req: Request, res: Response) => { - /** - * [P2-B] 快速 ACK 模式 - * 1. 仅做轻量校验和数据规范化(同步、无 I/O) - * 2. 将任务入队到 callbackQueue - * 3. 立即返回 202 Accepted,Jenkins 不会超时重试 - * 4. 后台 worker 异步消费队列,执行 completeBatchExecution + releaseSlot - */ - const receiveTimeMs = Date.now(); - const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; - - const { - runId, - status, - passedCases: reportedPassedCases = 0, - failedCases: reportedFailedCases = 0, - skippedCases: reportedSkippedCases = 0, - durationMs = 0, - results = [], - // 轻量化回调模式:仅发送 buildNumber - buildNumber, - } = req.body; - - // 判断是否为轻量化回调(有 buildNumber 但无 results) - const isLightweightCallback = !Array.isArray(results) || results.length === 0; - - const rawResults = Array.isArray(results) ? results : []; - const normalizedResults = normalizeCallbackResults(rawResults); - let passedCases = typeof reportedPassedCases === 'number' ? reportedPassedCases : 0; - let failedCases = typeof reportedFailedCases === 'number' ? reportedFailedCases : 0; - let skippedCases = typeof reportedSkippedCases === 'number' ? reportedSkippedCases : 0; - - // 从详细结果推导计数(与旧逻辑一致) - if (normalizedResults.length > 0) { - let derivedPassed = 0; - let derivedFailed = 0; - let derivedSkipped = 0; - - for (const result of normalizedResults) { - const caseStatus = String(result['status'] || '').toLowerCase(); - if (caseStatus === 'passed') derivedPassed++; - else if (caseStatus === 'failed' || caseStatus === 'error') derivedFailed++; - else derivedSkipped++; - } - - const totalReported = passedCases + failedCases + skippedCases; - const totalDerived = derivedPassed + derivedFailed + derivedSkipped; - const shouldUseDerived = totalReported === 0 - || totalReported !== normalizedResults.length - || totalReported !== totalDerived; - - if (shouldUseDerived) { - logger.warn('Callback summary mismatch, using derived counts', { - runId, - reported: { passedCases, failedCases, skippedCases, total: totalReported }, - derived: { passedCases: derivedPassed, failedCases: derivedFailed, skippedCases: derivedSkipped, total: totalDerived }, - resultsCount: normalizedResults.length, - }, LOG_CONTEXTS.JENKINS); - passedCases = derivedPassed; - failedCases = derivedFailed; - skippedCases = derivedSkipped; - } - } - - // 规范化状态值 - const normalizedReportedStatus = normalizeCallbackTerminalStatus(status); - - if (normalizedReportedStatus !== status) { - logger.warn('Invalid callback status, treating as failed', { - runId, - providedStatus: status, - validStatuses: CALLBACK_TERMINAL_STATUSES, - }, LOG_CONTEXTS.JENKINS); - } - - const hasCallbackSummary = (passedCases + failedCases + skippedCases) > 0; - const normalizedStatus = hasCallbackSummary - ? deriveCallbackTerminalStatus({ - reportedStatus: normalizedReportedStatus, - passedCases, - failedCases, - skippedCases, - }) - : normalizedReportedStatus; - - logger.info('Jenkins callback received, enqueuing for async processing', { - runId, - status: normalizedStatus, - passedCases, - failedCases, - skippedCases, - durationMs, - resultsCount: normalizedResults.length, - clientIP, - userAgent: req.get('User-Agent'), - receiveTimeMs, - isLightweightCallback, - buildNumber, - }, LOG_CONTEXTS.JENKINS); - - // 入队(非阻塞) - const enqueued = callbackQueue.enqueue({ - runId, - status: normalizedStatus, - passedCases, - failedCases, - skippedCases, - durationMs, - results: normalizedResults, - // 轻量化回调参数 - buildNumber: isLightweightCallback ? buildNumber : undefined, - needsServerParsing: isLightweightCallback, - }); - - if (!enqueued) { - // 队列已满:返回 429 让 Jenkins 稍后重试 - rateLimitMiddleware.increment429Count(); - logger.error('Callback queue full, returning 429', { - event: LOG_EVENTS.JENKINS_CALLBACK_QUEUE_FULL, - runId, - queueMetrics: callbackQueue.getMetrics(), - }, LOG_CONTEXTS.JENKINS); - return res.status(429).json({ - success: false, - message: 'Callback queue is full. Please retry later.', - retryAfter: 5, - }); - } - - // 快速 ACK(202 Accepted:已接受,正在异步处理) - const ackTimeMs = Date.now() - receiveTimeMs; - return res.status(202).json({ - success: true, - message: 'Callback accepted for async processing', - ackTimeMs, - }); -}); - -/** - * GET /api/jenkins/batch/:runId - * 获取执行批次详情 - */ -router.get('/batch/:runId', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - try { - const runId = parseInt(req.params.runId); - if (isNaN(runId) || runId <= 0) { - return res.status(400).json({ - success: false, - message: 'Invalid runId parameter. Must be a positive integer.' - }); - } - const batch = await executionService.getBatchExecution(runId); - - const e = batch.execution; - - // 将 TypeORM entity 的 camelCase 字段映射为 snake_case,与前端 TestRunRecord 接口对齐 - res.json({ - success: true, - data: { - id: e.id, - project_id: e.projectId ?? null, - project_name: null, - status: e.status, - trigger_type: e.triggerType, - trigger_by: e.triggerBy, - trigger_by_name: e.triggerByName ?? null, - jenkins_job: e.jenkinsJob ?? null, - jenkins_build_id: e.jenkinsBuildId ?? null, - jenkins_url: e.jenkinsUrl ?? null, - total_cases: e.totalCases, - passed_cases: e.passedCases, - failed_cases: e.failedCases, - skipped_cases: e.skippedCases, - duration_ms: e.durationMs, - start_time: e.startTime ?? null, - end_time: e.endTime ?? null, - created_at: e.createdAt, - } - }); - } catch (error: unknown) { - logger.errorLog(error, 'Failed to get batch execution', { - event: LOG_EVENTS.JENKINS_CALLBACK_FAILED, - runId: req.params.runId, - }); - const message = error instanceof Error ? error.message : 'Unknown error'; - res.status(500).json({ success: false, message }); - } -}); - -/** - * POST /api/jenkins/callback/test - * 测试回调连接 - 支持传入真实数据进行测试处理 - * 可选参数: runId, status, passedCases, failedCases, skippedCases, durationMs, results - * 如果提供了 runId,则会真实处理回调数据;否则仅测试连接 - * 通过 IP 白名单验证 - */ -router.post('/callback/test', [ - ipWhitelistMiddleware.verify, - rateLimitMiddleware.limit -], async (req: Request, res: Response) => { - const startTime = Date.now(); - try { - const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; - const timestamp = new Date().toISOString(); - - // 检查是否提供了真实的回调数据 - const { - testMessage = 'test', - runId, - status, - passedCases, - failedCases, - skippedCases, - durationMs, - results - } = req.body; - - const isRealDataTest = !!runId && !!status; - const normalizedInputResults = Array.isArray(results) ? normalizeCallbackResults(results) : []; - - logger.debug(`Received test callback from ${clientIP}`, { - timestamp, - isRealDataTest, - runId, - status, - dataMode: isRealDataTest ? 'REAL_DATA' : 'CONNECTION_TEST', - headers: { - contentType: req.headers['content-type'], - }, - clientIP, - }, LOG_CONTEXTS.JENKINS); - - // 如果提供了真实回调数据,则处理它 - if (isRealDataTest) { - logger.info(`Processing real callback test data`, { - runId, - status, - passedCases: passedCases || 0, - failedCases: failedCases || 0, - skippedCases: skippedCases || 0, - durationMs: durationMs || 0, - resultsCount: normalizedInputResults.length - }, LOG_CONTEXTS.JENKINS); - try { - // 真实处理回调 - await executionService.completeBatchExecution(runId, { - status: status || 'failed', - passedCases: passedCases || 0, - failedCases: failedCases || 0, - skippedCases: skippedCases || 0, - durationMs: durationMs || 0, - results: normalizedInputResults, - }); - const processingTime = Date.now() - startTime; - - logger.info(`Successfully processed real callback test data for runId ${runId}`, { - runId, - processingTimeMs: processingTime, - dataMode: 'REAL_DATA', - }, LOG_CONTEXTS.JENKINS); - - res.json({ - success: true, - message: 'Test callback processed successfully - 测试回调数据已处理', - mode: 'REAL_DATA', - details: { - receivedAt: timestamp, - clientIP, - testMessage, - processedData: { - runId, - status, - passedCases: passedCases || 0, - failedCases: failedCases || 0, - skippedCases: skippedCases || 0, - durationMs: durationMs || 0, - resultsCount: normalizedInputResults.length - } - }, - diagnostics: { - platform: process.env.NODE_ENV, - jenkinsUrl: process.env.JENKINS_URL, - callbackReceived: true, - networkConnectivity: 'OK', - dataProcessing: 'SUCCESS', - timestamp, - processingTimeMs: processingTime - }, - recommendations: [ - '✅ 网络连接正常', - '✅ 回调数据已成功处理', - '✅ 可以开始集成 Jenkins' - ] - }); - } catch (processError) { - const errorMessage = processError instanceof Error ? processError.message : 'Unknown error'; - const processingTime = Date.now() - startTime; - - logger.error(`Failed to process real callback test data for runId ${runId}`, { - event: LOG_EVENTS.JENKINS_CALLBACK_TEST_FAILED, - runId, - error: errorMessage, - stack: processError instanceof Error ? processError.stack : undefined, - processingTimeMs: processingTime - }, LOG_CONTEXTS.JENKINS); - - res.status(500).json({ - success: false, - message: `Failed to process callback data: ${errorMessage}`, - mode: 'REAL_DATA', - details: { - error: errorMessage, - timestamp: new Date().toISOString(), - runId, - processingTimeMs: processingTime, - suggestions: [ - '检查 runId 是否存在于数据库', - '查看后端日志获取详细错误信息', - '确保所有必需字段都已提供' - ] - } - }); - } - } else { - // 仅测试连接 - res.json({ - success: true, - message: 'Callback test successful - 回调连接测试通过', - mode: 'CONNECTION_TEST', - details: { - receivedAt: timestamp, - clientIP, - testMessage, - }, - diagnostics: { - platform: process.env.NODE_ENV, - jenkinsUrl: process.env.JENKINS_URL, - callbackReceived: true, - networkConnectivity: 'OK', - timestamp, - }, - recommendations: [ - '✅ 网络连接正常', - '✅ 可以开始集成 Jenkins', - '💡 提示:可以传入 runId、status 等参数来测试真实回调处理' - ] - }); - } - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error(`Test callback failed`, { - event: LOG_EVENTS.JENKINS_CALLBACK_FAILED, - error: message, - stack: error instanceof Error ? error.stack : undefined, - timestamp: new Date().toISOString() - }, LOG_CONTEXTS.JENKINS); - res.status(500).json({ - success: false, - message, - details: { - error: message, - timestamp: new Date().toISOString(), - suggestions: [ - '检查请求头中的认证信息', - '验证 IP 地址是否在白名单中', - '确保请求格式正确' - ] - } - }); - } -}); - -/** - * POST /api/jenkins/callback/manual-sync/:runId - * 手动同步执行状态 - 用于修复卡住的运行记录 - * 从数据库查询当前状态并允许手动更新 - * 通过 IP 白名单验证 - */ -router.post('/callback/manual-sync/:runId', [ - ipWhitelistMiddleware.verify, - rateLimitMiddleware.limit -], async (req: Request, res: Response) => { - try { - const runId = parseInt(req.params.runId); - const syncBody = (req.body ?? {}) as Record; - const status = syncBody['status']; - const passedCases = syncBody['passedCases']; - const failedCases = syncBody['failedCases']; - const skippedCases = syncBody['skippedCases']; - const durationMs = syncBody['durationMs']; - const results = syncBody['results']; - const force = typeof syncBody['force'] === 'boolean' ? syncBody['force'] : false; - const normalizedManualResults = Array.isArray(results) ? normalizeCallbackResults(results) : []; - - if (isNaN(runId) || runId <= 0) { - return res.status(400).json({ - success: false, - message: 'Invalid runId parameter. Must be a positive integer.' - }); - } - - logger.info(`Starting manual sync for execution`, { - runId, - status, - passedCases, - failedCases, - skippedCases, - durationMs, - resultsCount: normalizedManualResults.length, - force, - timestamp: new Date().toISOString() - }, LOG_CONTEXTS.JENKINS); - - // 查询现有运行记录 - const execution = await executionService.getBatchExecution(runId); - - if (!execution.execution) { - return res.status(404).json({ - success: false, - message: `Execution not found: runId=${runId}` - }); - } - - const executionData = execution.execution as unknown as Record; - const currentStatus = executionData['status']; - - // 检查是否允许更新 - if (!force && ['success', 'failed', 'cancelled'].includes(currentStatus as string)) { - return res.status(400).json({ - success: false, - message: `Execution is already completed with status: ${currentStatus}. Use force=true to override.`, - current: { - id: runId, - status: currentStatus, - totalCases: executionData['total_cases'], - passedCases: executionData['passed_cases'], - failedCases: executionData['failed_cases'], - skippedCases: executionData['skipped_cases'], - updatedAt: executionData['updated_at'] ?? executionData['created_at'] - } - }); - } - - // 必须提供新状态 - if (!status) { - return res.status(400).json({ - success: false, - message: 'status field is required for manual sync' - }); - } - - // 执行更新 - const startTime = Date.now(); - - await executionService.completeBatchExecution(runId, { - status: status as 'success' | 'failed' | 'cancelled', - passedCases: typeof passedCases === 'number' ? passedCases : 0, - failedCases: typeof failedCases === 'number' ? failedCases : 0, - skippedCases: typeof skippedCases === 'number' ? skippedCases : 0, - durationMs: typeof durationMs === 'number' ? durationMs : 0, - results: normalizedManualResults, - }); - - const processingTime = Date.now() - startTime; - - logger.info(`Successfully completed manual sync for execution`, { - runId, - processingTimeMs: processingTime, - timestamp: new Date().toISOString(), - }, LOG_CONTEXTS.JENKINS); - - // 查询更新后的数据 - const updated = await executionService.getBatchExecution(runId); - - const updatedData = updated.execution as unknown as Record; - - res.json({ - success: true, - message: 'Manual sync completed successfully', - previous: { - id: runId, - status: currentStatus, - totalCases: executionData['total_cases'], - passedCases: executionData['passed_cases'], - failedCases: executionData['failed_cases'], - skippedCases: executionData['skipped_cases'] - }, - updated: { - id: runId, - status: updatedData['status'], - totalCases: updatedData['total_cases'], - passedCases: updatedData['passed_cases'], - failedCases: updatedData['failed_cases'], - skippedCases: updatedData['skipped_cases'], - endTime: updatedData['end_time'], - durationMs: updatedData['duration_ms'] - }, - timing: { - processingTimeMs: processingTime, - timestamp: new Date().toISOString() - } - }); - - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - const errorDetails = error instanceof Error ? error.stack : undefined; - - logger.error(`Failed to complete manual sync for execution`, { - event: LOG_EVENTS.JENKINS_MANUAL_SYNC_FAILED, - runId: req.params.runId, - error: message, - stack: errorDetails, - timestamp: new Date().toISOString() - }, LOG_CONTEXTS.JENKINS); - - res.status(500).json({ - success: false, - message: `Manual sync failed: ${message}`, - details: { - error: message, - timestamp: new Date().toISOString(), - suggestions: [ - '检查 runId 是否存在于数据库', - '确保传入的状态值有效(success、failed、aborted)', - '查看后端日志获取详细错误信息', - '如果执行已完成,使用 force=true 强制更新' - ] - } - }); - } -}); - -/** - * POST /api/jenkins/callback/diagnose - * 诊断回调连接问题 - 通过 IP 白名单验证以保护系统信息 - * - * 安全建议:建议添加管理员权限验证 - * TODO: 添加 requireAuth 和 requireRole('admin') 中间件以增强安全性 - */ -router.post('/callback/diagnose', - generalAuthRateLimiter, - optionalAuth, // 添加可选认证,获取用户信息 - rateLimitMiddleware.limit, - ipWhitelistMiddleware.verify, - async (req: Request, res: Response) => { - // 检查用户权限(如果已认证) - if (req.user && process.env.NODE_ENV === 'production') { - // 在生产环境中,建议检查用户是否为管理员 - // if (req.user.role !== 'admin') { - // return res.status(403).json({ - // success: false, - // message: 'Access denied. Admin privileges required.' - // }); - // } - logger.info('Diagnostic request from authenticated user', { - userId: req.user.id, - userEmail: req.user.email, - }, LOG_CONTEXTS.JENKINS); - } - try { - const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; - const timestamp = new Date().toISOString(); - - logger.debug(`Received callback diagnostic request`, { - clientIP, - timestamp, - headers: Object.keys(req.headers).filter(k => k.toLowerCase().includes('auth') || k.toLowerCase().includes('jenkins')) - }, LOG_CONTEXTS.JENKINS); - - // 分析回调配置 - const envConfig = { - jenkins_url: !!process.env.JENKINS_URL, - jenkins_user: !!process.env.JENKINS_USER, - jenkins_token: !!process.env.JENKINS_TOKEN, - jenkins_allowed_ips: !!process.env.JENKINS_ALLOWED_IPS, - }; - const diagnostics: { - timestamp: string; - clientIP: string; - environmentVariablesConfigured: typeof envConfig; - requestHeaders: Record; - suggestions: string[]; - nextSteps?: string[]; - } = { - timestamp, - clientIP, - environmentVariablesConfigured: envConfig, - requestHeaders: { - hasContentType: !!req.headers['content-type'], - }, - suggestions: [], - }; - - // 分析问题并给出建议 - if (!diagnostics.environmentVariablesConfigured.jenkins_token) { - diagnostics.suggestions.push('⚠️ 未配置 JENKINS_TOKEN,Jenkins API 集成可能无法正常工作'); - } - if (!diagnostics.environmentVariablesConfigured.jenkins_allowed_ips) { - diagnostics.suggestions.push('⚠️ 未配置 JENKINS_ALLOWED_IPS,将允许所有 IP 访问回调接口'); - } - - if (diagnostics.suggestions.length === 0) { - diagnostics.suggestions.push('✅ 所有必需的环境变量已配置'); - diagnostics.suggestions.push('✅ 回调接口已就绪'); - } - - // 提供配置步骤 - diagnostics.nextSteps = [ - '1️⃣ 配置 JENKINS_ALLOWED_IPS 以限制回调源 IP(推荐)', - '2️⃣ 配置 JENKINS_URL、JENKINS_USER、JENKINS_TOKEN 用于 API 集成', - '3️⃣ 使用 curl 测试回调:', - ' curl -X POST http://localhost:3000/api/jenkins/callback/test \\', - ' -H "Content-Type: application/json" \\', - ' -d \'{"testMessage": "hello"}\'', - '4️⃣ 如果收到成功响应,可以开始集成 Jenkins', - '📚 详细文档:docs/JENKINS_CONFIG_GUIDE.md' - ]; - - res.json({ - success: true, - data: diagnostics, - message: 'Diagnostic report generated' - }); - - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error(`Callback diagnostic failed`, { - event: LOG_EVENTS.JENKINS_DIAGNOSE_FAILED, - error: message, - }, LOG_CONTEXTS.JENKINS); - res.status(500).json({ - success: false, - message: `Diagnostic failed: ${message}` - }); - } -}); - -/** - * GET /api/jenkins/health - * Jenkins 连接健康检查 - 包括详细的诊断信息 - */ -router.get('/health', generalAuthRateLimiter, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - const startTime = Date.now(); - - try { - logger.info(`Starting Jenkins health check...`, {}, LOG_CONTEXTS.JENKINS); - - // 测试 Jenkins 连接 - // 生产环境强制要求配置 Jenkins 环境变量 - if (process.env.NODE_ENV === 'production') { - if (!process.env.JENKINS_URL || !process.env.JENKINS_USER || !process.env.JENKINS_TOKEN) { - return res.status(500).json({ - success: false, - message: 'Jenkins configuration is missing in production environment', - data: { - connected: false, - details: { - issues: [ - '❌ 生产环境缺少必需的 Jenkins 配置', - !process.env.JENKINS_URL ? '❌ JENKINS_URL 未配置' : '', - !process.env.JENKINS_USER ? '❌ JENKINS_USER 未配置' : '', - !process.env.JENKINS_TOKEN ? '❌ JENKINS_TOKEN 未配置' : '', - ].filter(Boolean), - recommendations: [ - '请在环境变量中配置 JENKINS_URL', - '请在环境变量中配置 JENKINS_USER', - '请在环境变量中配置 JENKINS_TOKEN', - ], - }, - }, - }); - } - } - - const jenkinsUrl = process.env.JENKINS_URL || DEFAULT_JENKINS_URL; - const jenkinsUser = process.env.JENKINS_USER || DEFAULT_JENKINS_USER; - const jenkinsToken = process.env.JENKINS_TOKEN || ''; - - // 健康检查数据 - const healthCheckData: { - timestamp: string; - duration: number; - checks: Record; - diagnostics: Record; - issues: string[]; - recommendations: string[]; - } = { - timestamp: new Date().toISOString(), - duration: 0, - checks: { - connectionTest: { success: false, duration: 0 }, - authenticationTest: { success: false, duration: 0 }, - apiResponseTest: { success: false, duration: 0 }, - targetJobInspection: { success: false, duration: 0 }, - }, - diagnostics: { - configPresent: { - url: !!jenkinsUrl, - user: !!jenkinsUser, - token: !!jenkinsToken, - } - }, - issues: [] as string[], - recommendations: [] as string[], - }; - - // 1. 测试基础连接 - logger.debug(`Testing connection to Jenkins`, { - jenkinsUrl, - }, LOG_CONTEXTS.JENKINS); - const connStartTime = Date.now(); - - // 构建 API URL(处理 URL 尾部斜杠) - let apiUrl = jenkinsUrl; - if (!apiUrl.endsWith('/')) { - apiUrl += '/'; - } - apiUrl += 'api/json'; - - logger.debug(`Final API URL for health check`, { - apiUrl, - }, LOG_CONTEXTS.JENKINS); - - const credentials = Buffer.from(`${jenkinsUser}:${jenkinsToken}`).toString('base64'); - - // 设置超时 - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS); - - try { - const response = await fetch(apiUrl, { - method: 'GET', - headers: { - 'Authorization': `Basic ${credentials}`, - 'Content-Type': 'application/json', - }, - signal: controller.signal, - }); - - clearTimeout(timeoutId); - healthCheckData.checks.connectionTest.duration = Date.now() - connStartTime; - healthCheckData.checks.connectionTest.success = response.ok; - healthCheckData.diagnostics.connectionStatus = response.status; - healthCheckData.diagnostics.statusText = response.statusText; - - logger.debug(`Jenkins health check response received`, { - status: response.status, - statusText: response.statusText, - duration: healthCheckData.checks.connectionTest.duration, - }, LOG_CONTEXTS.JENKINS); - - if (response.ok) { - const data = await response.json() as Record; - healthCheckData.checks.authenticationTest.success = true; - healthCheckData.checks.apiResponseTest.success = true; - let triggerReady = false; - - const targetJobStart = Date.now(); - try { - const targetJobInspection = await jenkinsService.inspectConfiguredApiJob(); - healthCheckData.checks.targetJobInspection.duration = Date.now() - targetJobStart; - healthCheckData.checks.targetJobInspection.success = Boolean(targetJobInspection?.triggerReady); - healthCheckData.diagnostics.targetJobInspection = targetJobInspection; - triggerReady = Boolean(targetJobInspection?.triggerReady); - - if (targetJobInspection) { - healthCheckData.issues.push(...targetJobInspection.issues); - healthCheckData.recommendations.push(...targetJobInspection.recommendations); - } - } catch (inspectionError) { - healthCheckData.checks.targetJobInspection.duration = Date.now() - targetJobStart; - healthCheckData.diagnostics.targetJobInspectionError = - inspectionError instanceof Error ? inspectionError.message : String(inspectionError); - healthCheckData.issues.push('❌ 无法读取目标 Jenkins Job 的实时配置'); - healthCheckData.recommendations.push('检查 Jenkins Job 权限,确保当前账号具备读取任务配置的权限。'); - } - - healthCheckData.duration = Date.now() - startTime; - - res.json({ - success: true, - data: { - connected: true, - triggerReady, - jenkinsUrl, - version: typeof data['version'] === 'string' ? data['version'] : 'unknown', - timestamp: new Date().toISOString(), - details: healthCheckData, - }, - message: triggerReady - ? 'Jenkins is healthy' - : 'Jenkins is reachable, but the target job needs configuration fixes before the platform can trigger it' - }); - } else if (response.status === 401 || response.status === 403) { - healthCheckData.issues.push('❌ 认证失败:API Token 或用户名可能不正确'); - healthCheckData.recommendations.push('检查 JENKINS_USER 和 JENKINS_TOKEN 环境变量'); - - res.status(response.status).json({ - success: false, - data: { - connected: false, - status: response.status, - statusText: response.statusText, - details: healthCheckData, - }, - message: 'Jenkins service authentication failed. Please check configuration.' - }); - } else { - healthCheckData.issues.push(`❌ Jenkins 返回错误状态: ${response.status} ${response.statusText}`); - healthCheckData.recommendations.push('检查 Jenkins 服务是否正常运行'); - healthCheckData.recommendations.push('检查 JENKINS_URL 是否正确'); - - res.status(response.status).json({ - success: false, - data: { - connected: false, - status: response.status, - statusText: response.statusText, - details: healthCheckData, - }, - message: `Jenkins returned ${response.status}: ${response.statusText}` - }); - } - } catch (fetchError) { - clearTimeout(timeoutId); - - const fetchErrorMsg = fetchError instanceof Error ? fetchError.message : String(fetchError); - healthCheckData.checks.connectionTest.duration = Date.now() - connStartTime; - - if (fetchErrorMsg.includes('ECONNREFUSED')) { - healthCheckData.issues.push('❌ 连接被拒绝:Jenkins 服务可能未运行'); - healthCheckData.recommendations.push('确保 Jenkins 服务已启动'); - } else if (fetchErrorMsg.includes('ENOTFOUND')) { - healthCheckData.issues.push('❌ DNS 解析失败:无法解析 Jenkins 域名'); - healthCheckData.recommendations.push('检查 JENKINS_URL 中的域名是否正确'); - healthCheckData.recommendations.push('检查网络连接和 DNS 配置'); - } else if (fetchErrorMsg.includes('Aborted')) { - healthCheckData.issues.push('❌ 请求超时:Jenkins 响应时间过长(> 10秒)'); - healthCheckData.recommendations.push('检查 Jenkins 服务状态和网络连接'); - healthCheckData.recommendations.push('考虑增加超时时间'); - } else { - healthCheckData.issues.push(`❌ 网络错误:${fetchErrorMsg}`); - } - - throw fetchError; - } - } catch (error: unknown) { - const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_HEALTH'); - - res.status(500).json({ - success: false, - data: { - connected: false, - error: sanitizedMessage, - details: { - timestamp: new Date().toISOString(), - duration: Date.now() - startTime, - issues: [ - '❌ 无法连接到 Jenkins', - '请检查Jenkins服务状态和网络连接' - ], - recommendations: [ - '检查 Jenkins 服务是否运行', - '检查网络连接', - '验证 Jenkins URL 配置', - '查看应用日志获取详细错误信息' - ] - }, - stack: process.env.NODE_ENV === 'development' ? (error instanceof Error ? error.stack : undefined) : undefined - }, - message: `Failed to connect to Jenkins: ${sanitizedMessage}` - }); - } -}); - -/** - * GET /api/jenkins/diagnose - * 诊断执行问题 - 通过 IP 白名单验证以保护系统信息 - */ -router.get('/diagnose', - generalAuthRateLimiter, - rateLimitMiddleware.limit, - ipWhitelistMiddleware.verify, - async (req: Request, res: Response) => { - try { - const runId = parseInt(req.query.runId as string); - - if (isNaN(runId) || runId <= 0) { - return res.status(400).json({ - success: false, - message: 'Invalid runId parameter. Must be a positive integer.' - }); - } - - logger.info(`Starting execution diagnosis`, { - runId, - }, LOG_CONTEXTS.JENKINS); - - // 获取执行批次信息 - const batch = await executionService.getBatchExecution(runId); - const execution = batch.execution; - - // 计算执行时长 - const startTime = execution.startTime ? new Date(execution.startTime).getTime() : null; - const currentTime = Date.now(); - const executionDuration = startTime ? currentTime - startTime : 0; - - // 检查Jenkins连接状态 - let jenkinsConnectivity: any = null; - if (execution.jenkinsJob && execution.jenkinsBuildId) { - try { - const buildStatus = await jenkinsStatusService.getBuildStatus( - execution.jenkinsJob as string, - execution.jenkinsBuildId as string - ); - jenkinsConnectivity = { - canConnect: !!buildStatus, - buildStatus: buildStatus ? { - building: buildStatus.building, - result: buildStatus.result, - duration: buildStatus.duration, - url: buildStatus.url - } : null - }; - } catch (error) { - jenkinsConnectivity = { - canConnect: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - // 收集诊断信息 - const diagnostics = { - executionId: execution.id, - status: execution.status, - jenkinsJob: execution.jenkinsJob, - jenkinsBuildId: execution.jenkinsBuildId, - jenkinsUrl: execution.jenkinsUrl, - startTime: execution.startTime, - createdAt: execution.createdAt, - totalCases: execution.totalCases, - passedCases: execution.passedCases, - failedCases: execution.failedCases, - skippedCases: execution.skippedCases, - executionDuration, - - // 诊断信息 - diagnostics: { - jenkinsInfoMissing: !execution.jenkinsJob || !execution.jenkinsBuildId || !execution.jenkinsUrl, - startTimeMissing: !execution.startTime, - stillPending: execution.status === 'pending', - stillRunning: execution.status === 'running', - noTestResults: execution.passedCases === 0 && execution.failedCases === 0 && execution.skippedCases === 0, - longRunning: executionDuration > 5 * 60 * 1000, // 超过5分钟 - veryLongRunning: executionDuration > 10 * 60 * 1000, // 超过10分钟 - jenkinsConnectivity, - - // 时间分析 - timeAnalysis: { - executionAge: executionDuration, - executionAgeMinutes: Math.round(executionDuration / 60000), - isOld: executionDuration > 30 * 60 * 1000, // 超过30分钟 - createdRecently: startTime && execution.createdAt ? (currentTime - new Date(execution.createdAt).getTime()) < 60 * 1000 : false - }, - - // 建议 - suggestions: [] as string[] - } - }; - - // 生成建议 - const sugg = diagnostics.diagnostics.suggestions; - - if (diagnostics.diagnostics.jenkinsInfoMissing) { - sugg.push('🚨 Jenkins 信息未被填充。这通常表示 Jenkins 触发失败。请检查后端日志查找错误信息。'); - } - - if (diagnostics.diagnostics.startTimeMissing) { - sugg.push('⏳ 执行开始时间为空。这表示 Jenkins 尚未开始构建。请等待几秒后重试。'); - } - - if (diagnostics.diagnostics.stillPending) { - if (diagnostics.diagnostics.timeAnalysis.executionAgeMinutes > 2) { - sugg.push('⚠️ 执行已处于 pending 状态超过2分钟,可能存在问题。建议手动同步状态。'); - } else { - sugg.push('⏳ 执行仍处于 pending 状态。这是正常的,系统正在等待 Jenkins 接收任务。'); - } - } - - if (diagnostics.diagnostics.stillRunning) { - if (diagnostics.diagnostics.veryLongRunning) { - sugg.push('🚨 执行已运行超过10分钟,可能卡住了。建议检查Jenkins构建状态或手动同步。'); - } else if (diagnostics.diagnostics.longRunning) { - sugg.push('⚠️ 执行已运行超过5分钟,请检查是否正常。可以尝试手动同步状态。'); - } - } - - if (diagnostics.diagnostics.noTestResults && !diagnostics.diagnostics.stillPending) { - sugg.push('❌ 测试结果为空。这可能表示 Jenkins 任务失败或回调未到达。请检查 Jenkins 的执行日志。'); - } - - // Jenkins连接性建议 - if (jenkinsConnectivity) { - if (!jenkinsConnectivity.canConnect) { - sugg.push('🔌 无法连接到Jenkins获取构建状态。请检查Jenkins服务器状态和网络连接。'); - } else if (jenkinsConnectivity.buildStatus) { - const buildStatus = jenkinsConnectivity.buildStatus; - if (!buildStatus.building && buildStatus.result) { - if (execution.status === 'running') { - sugg.push(`🔄 Jenkins显示构建已完成(${buildStatus.result}),但平台状态仍为running。建议立即手动同步。`); - } - } - } - } - - // 基于时间的建议 - if (diagnostics.diagnostics.timeAnalysis.isOld) { - sugg.push('🕐 执行时间过长(超过30分钟),建议检查或取消该执行。'); - } - - if (sugg.length === 0) { - sugg.push('✅ 执行状态良好,无明显问题。'); - } - - res.json({ - success: true, - data: diagnostics - }); - - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error(`Execution diagnosis failed`, { - event: LOG_EVENTS.JENKINS_DIAGNOSE_FAILED, - error: message, - }, LOG_CONTEXTS.JENKINS); - res.status(500).json({ - success: false, - message: `Diagnosis failed: ${message}` - }); - } -}); - -/** - * GET /api/jenkins/monitoring/stats - * 获取监控统计信息 - */ -router.get('/monitoring/stats', generalAuthRateLimiter, rateLimitMiddleware.limit, async (_req, res) => { - try { - logger.info(`Getting monitoring statistics...`, {}, LOG_CONTEXTS.JENKINS); - - // 获取混合同步服务的统计信息 - const syncStats = hybridSyncService.getMonitoringStats(); - - // 获取最近的执行统计 - const recentExecutions = await executionService.getRecentExecutions(50) as any[]; - const statusCounts = recentExecutions.reduce((acc: Record, exec: any) => { - acc[exec.status] = (acc[exec.status] || 0) + 1; - return acc; - }, {}); - - // 计算卡住的执行数量 - const stuckExecutions = recentExecutions.filter((exec: any) => { - if (!['running', 'pending'].includes(exec.status) || !exec.start_time) return false; - const duration = Date.now() - new Date(exec.start_time).getTime(); - return duration > 5 * 60 * 1000; // 超过5分钟 - }); - - const stats = { - timestamp: new Date().toISOString(), - syncService: syncStats, - executions: { - total: recentExecutions.length, - byStatus: statusCounts, - stuck: stuckExecutions.length, - stuckList: stuckExecutions.map((exec: any) => ({ - id: exec.id, - status: exec.status, - duration: Date.now() - new Date(exec.start_time).getTime(), - jenkins_job: exec.jenkins_job, - jenkins_build_id: exec.jenkins_build_id - })) - }, - health: { - totalIssues: syncStats.failed + syncStats.timeout + stuckExecutions.length, - hasIssues: (syncStats.failed + syncStats.timeout + stuckExecutions.length) > 0 - } - }; - - res.json({ - success: true, - data: stats - }); - - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error(`Failed to get monitoring statistics`, { - event: LOG_EVENTS.JENKINS_MONITORING_STATS_FAILED, - error: message, - }, LOG_CONTEXTS.JENKINS); - res.status(500).json({ - success: false, - message: `Failed to get monitoring stats: ${message}` - }); - } -}); - -/** - * POST /api/jenkins/monitoring/fix-stuck - * 修复卡住的执行 - */ -router.post('/monitoring/fix-stuck', generalAuthRateLimiter, rateLimitMiddleware.limit, async (req: Request, res: Response) => { - try { - const fixBody = (req.body ?? {}) as Record; - const timeoutMinutes = typeof fixBody['timeoutMinutes'] === 'number' ? fixBody['timeoutMinutes'] : 5; - const dryRun = typeof fixBody['dryRun'] === 'boolean' ? fixBody['dryRun'] : false; - - logger.info(`${dryRun ? 'Simulating' : 'Starting'} fix for stuck executions`, { - timeoutMinutes, - dryRun, - }, LOG_CONTEXTS.JENKINS); - - if (dryRun) { - // 只查询,不修复 - const timeoutMs = timeoutMinutes * 60 * 1000; - const timeoutThreshold = new Date(Date.now() - timeoutMs); - - const stuckExecutions = await query(` - SELECT id, status, jenkins_job, jenkins_build_id, jenkins_url, - start_time, TIMESTAMPDIFF(MINUTE, start_time, NOW()) as duration_minutes - FROM Auto_TestRun - WHERE status IN ('pending', 'running') - AND start_time < ? - ORDER BY start_time ASC - LIMIT 20 - `, [timeoutThreshold]) as any[]; - - res.json({ - success: true, - data: { - dryRun: true, - wouldFix: stuckExecutions.length, - executions: stuckExecutions - } - }); - } else { - // 实际修复 - const timeoutMs = timeoutMinutes * 60 * 1000; - const result = await executionService.checkAndHandleTimeouts(timeoutMs); - - res.json({ - success: true, - data: { - dryRun: false, - checked: result.checked, - updated: result.updated, - timedOut: result.timedOut, - message: `Fixed ${result.updated} executions, marked ${result.timedOut} as timed out` - } - }); - } - - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error(`Failed to fix stuck executions`, { - event: LOG_EVENTS.JENKINS_FIX_STUCK_FAILED, - error: message, - }, LOG_CONTEXTS.JENKINS); - res.status(500).json({ - success: false, - message: `Failed to fix stuck executions: ${message}` - }); - } -}); - -/** - * GET /api/jenkins/monitor/status - * Get execution monitor service status and statistics - */ -router.get('/monitor/status', generalAuthRateLimiter, rateLimitMiddleware.limit, async (_req: Request, res: Response) => { - try { - const status = executionMonitorService.getStatus(); - const stats = executionMonitorService.getStats(); - - logger.debug('Monitor status requested', { - isRunning: status.isRunning, - cyclesRun: stats.cyclesRun, - }, LOG_CONTEXTS.MONITOR); - - res.json({ - success: true, - data: { - status: status.isRunning ? 'running' : 'stopped', - isRunning: status.isRunning, - config: status.config, - stats: { - cyclesRun: stats.cyclesRun, - totalExecutionsChecked: stats.totalExecutionsChecked, - totalExecutionsUpdated: stats.totalExecutionsUpdated, - totalCompilationFailures: stats.totalCompilationFailures, - totalErrors: stats.totalErrors, - lastCycleTime: stats.lastCycleTime, - lastCycleDuration: stats.lastCycleDuration, - isProcessing: stats.isProcessing, - }, - }, - }); - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error'; - logger.error('Failed to get monitor status', { - event: LOG_EVENTS.JENKINS_MONITOR_STATUS_FAILED, - error: message, - }, LOG_CONTEXTS.MONITOR); - res.status(500).json({ - success: false, - message: `Failed to get monitor status: ${message}`, - }); - } -}); - -/** - * GET /api/jenkins/metrics - * 获取 Jenkins 集成相关的所有监控指标(P2-C) - * - * 聚合指标: - * - rateLimit: 429 次数、每分钟 429 速率、活跃 IP 数 - * - callbackQueue: 队列深度、总入队/处理/失败数、平均排队时长、重试分布 - * - jenkinsQueue: queueId 轮询总次数、成功解析数、超时/取消数、平均/最大等待时长 - * - process: 内存使用、进程运行时长 - * - * 访问控制:需要认证(通过 optionalAuth 获取用户信息,如未认证则仅返回部分指标) - */ -router.get('/metrics', [generalAuthRateLimiter, optionalAuth], (_req: Request, res: Response) => { - try { - const rateLimitMetrics = rateLimitMiddleware.getMetrics(); - const queueMetrics = callbackQueue.getMetrics(); - const jenkinsQueueMetrics = jenkinsService.getQueueMetrics(); - const memUsage = process.memoryUsage(); - - res.json({ - success: true, - timestamp: new Date().toISOString(), - data: { - /** - * 速率限制指标 - */ - rateLimit: { - total429Count: rateLimitMetrics.total429Count, - rate429PerMinute: rateLimitMetrics.rate429PerMinute, - activeIPs: rateLimitMetrics.activeIPs, - }, - - /** - * 回调队列指标(P2-B) - */ - callbackQueue: { - queueDepth: queueMetrics.queueDepth, - workerBusy: queueMetrics.workerBusy, - totalEnqueued: queueMetrics.totalEnqueued, - totalProcessed: queueMetrics.totalProcessed, - totalFailed: queueMetrics.totalFailed, - avgWaitMs: queueMetrics.avgWaitMs, - maxWaitMs: queueMetrics.maxWaitMs, - retryDistribution: queueMetrics.retryDistribution, - // 最近 20 条排队时长样本(用于画趋势图) - recentWaitSamples: queueMetrics.waitTimeSamples.slice(-20), - }, - - /** - * Jenkins 构建队列指标(P2-A) - */ - jenkinsQueue: { - totalPolls: jenkinsQueueMetrics.totalPolls, - resolvedCount: jenkinsQueueMetrics.resolvedCount, - timeoutCount: jenkinsQueueMetrics.timeoutCount, - avgWaitMs: jenkinsQueueMetrics.avgWaitMs, - maxWaitMs: jenkinsQueueMetrics.maxWaitMs, - resolutionRate: jenkinsQueueMetrics.totalPolls > 0 - ? Math.round((jenkinsQueueMetrics.resolvedCount / jenkinsQueueMetrics.totalPolls) * 100) - : 0, - // 最近 20 条 Jenkins 队列等待时长样本 - recentWaitSamples: jenkinsQueueMetrics.waitTimeSamples.slice(-20), - }, - - /** - * 进程级指标 - */ - process: { - uptimeSeconds: Math.floor(process.uptime()), - memoryMB: { - rss: Math.round(memUsage.rss / 1024 / 1024), - heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024), - heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024), - }, - }, - }, - }); - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error'; - res.status(500).json({ success: false, message }); - } -}); +registerJenkinsExecutionRoutes(router); +registerJenkinsCallbackToolRoutes(router); +registerJenkinsDiagnosticRoutes(router); export default router; diff --git a/server/routes/jenkinsCallbackToolRoutes.ts b/server/routes/jenkinsCallbackToolRoutes.ts new file mode 100644 index 0000000..25323a5 --- /dev/null +++ b/server/routes/jenkinsCallbackToolRoutes.ts @@ -0,0 +1,485 @@ +import { Router, Request, Response } from 'express'; +import { In } from 'typeorm'; +import { executionService, type Auto_TestRunResultsInput } from '../services/ExecutionService'; +import { jenkinsService } from '../services/JenkinsService'; +import { jenkinsStatusService } from '../services/JenkinsStatusService'; +import { taskSchedulerService } from '../services/TaskSchedulerService'; +import { callbackQueue, type CallbackPayload } from '../services/CallbackQueue'; +import { ipWhitelistMiddleware, rateLimitMiddleware } from '../middleware/JenkinsAuthMiddleware'; +import { requestValidator } from '../middleware/RequestValidator'; +import { generalAuthRateLimiter } from '../middleware/authRateLimiter'; +import { optionalAuth } from '../middleware/auth'; +import logger from '../utils/logger'; +import { buildJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnostics'; +import { persistJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnosticArtifact'; +import { validateScriptPathsInTestRepo } from '../utils/testRepoScriptPathValidator'; +import { LOG_CONTEXTS, LOG_EVENTS, createTimer } from '../config/logging'; +import { AppDataSource, query, queryOne } from '../config/database'; +import { TestCase } from '../entities/TestCase'; +import { hybridSyncService } from '../services/HybridSyncService'; +import { executionMonitorService } from '../services/ExecutionMonitorService'; +import { + CALLBACK_TERMINAL_STATUSES, + deriveCallbackTerminalStatus, + normalizeCallbackTerminalStatus, +} from '../services/ExecutionService/callbackStatus'; +import { + buildCallbackUrl, + normalizeCallbackResults, + preflightExecutableScriptPaths, + recordTriggerFailure, + resolveExecutionBusinessError, + resolveScriptPaths, + runJenkinsTriggerPrecheck, + sanitizeErrorMessage, + scheduleCallbackFallbackSync, + warnIfCallbackUrlIsLocal, +} from './jenkinsRouteSupport'; + +export function registerJenkinsCallbackToolRoutes(router: Router): void { +router.post('/callback/test', [ + ipWhitelistMiddleware.verify, + rateLimitMiddleware.limit +], async (req: Request, res: Response) => { + const startTime = Date.now(); + try { + const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; + const timestamp = new Date().toISOString(); + + // 检查是否提供了真实的回调数据 + const { + testMessage = 'test', + runId, + status, + passedCases, + failedCases, + skippedCases, + durationMs, + results + } = req.body; + + const isRealDataTest = !!runId && !!status; + const normalizedInputResults = Array.isArray(results) ? normalizeCallbackResults(results) : []; + + logger.debug(`Received test callback from ${clientIP}`, { + timestamp, + isRealDataTest, + runId, + status, + dataMode: isRealDataTest ? 'REAL_DATA' : 'CONNECTION_TEST', + headers: { + contentType: req.headers['content-type'], + }, + clientIP, + }, LOG_CONTEXTS.JENKINS); + + // 如果提供了真实回调数据,则处理它 + if (isRealDataTest) { + logger.info(`Processing real callback test data`, { + runId, + status, + passedCases: passedCases || 0, + failedCases: failedCases || 0, + skippedCases: skippedCases || 0, + durationMs: durationMs || 0, + resultsCount: normalizedInputResults.length + }, LOG_CONTEXTS.JENKINS); + try { + // 真实处理回调 + await executionService.completeBatchExecution(runId, { + status: status || 'failed', + passedCases: passedCases || 0, + failedCases: failedCases || 0, + skippedCases: skippedCases || 0, + durationMs: durationMs || 0, + results: normalizedInputResults, + }); + const processingTime = Date.now() - startTime; + + logger.info(`Successfully processed real callback test data for runId ${runId}`, { + runId, + processingTimeMs: processingTime, + dataMode: 'REAL_DATA', + }, LOG_CONTEXTS.JENKINS); + + res.json({ + success: true, + message: 'Test callback processed successfully - 测试回调数据已处理', + mode: 'REAL_DATA', + details: { + receivedAt: timestamp, + clientIP, + testMessage, + processedData: { + runId, + status, + passedCases: passedCases || 0, + failedCases: failedCases || 0, + skippedCases: skippedCases || 0, + durationMs: durationMs || 0, + resultsCount: normalizedInputResults.length + } + }, + diagnostics: { + platform: process.env.NODE_ENV, + jenkinsUrl: process.env.JENKINS_URL, + callbackReceived: true, + networkConnectivity: 'OK', + dataProcessing: 'SUCCESS', + timestamp, + processingTimeMs: processingTime + }, + recommendations: [ + '✅ 网络连接正常', + '✅ 回调数据已成功处理', + '✅ 可以开始集成 Jenkins' + ] + }); + } catch (processError) { + const errorMessage = processError instanceof Error ? processError.message : 'Unknown error'; + const processingTime = Date.now() - startTime; + + logger.error(`Failed to process real callback test data for runId ${runId}`, { + event: LOG_EVENTS.JENKINS_CALLBACK_TEST_FAILED, + runId, + error: errorMessage, + stack: processError instanceof Error ? processError.stack : undefined, + processingTimeMs: processingTime + }, LOG_CONTEXTS.JENKINS); + + res.status(500).json({ + success: false, + message: `Failed to process callback data: ${errorMessage}`, + mode: 'REAL_DATA', + details: { + error: errorMessage, + timestamp: new Date().toISOString(), + runId, + processingTimeMs: processingTime, + suggestions: [ + '检查 runId 是否存在于数据库', + '查看后端日志获取详细错误信息', + '确保所有必需字段都已提供' + ] + } + }); + } + } else { + // 仅测试连接 + res.json({ + success: true, + message: 'Callback test successful - 回调连接测试通过', + mode: 'CONNECTION_TEST', + details: { + receivedAt: timestamp, + clientIP, + testMessage, + }, + diagnostics: { + platform: process.env.NODE_ENV, + jenkinsUrl: process.env.JENKINS_URL, + callbackReceived: true, + networkConnectivity: 'OK', + timestamp, + }, + recommendations: [ + '✅ 网络连接正常', + '✅ 可以开始集成 Jenkins', + '💡 提示:可以传入 runId、status 等参数来测试真实回调处理' + ] + }); + } + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`Test callback failed`, { + event: LOG_EVENTS.JENKINS_CALLBACK_FAILED, + error: message, + stack: error instanceof Error ? error.stack : undefined, + timestamp: new Date().toISOString() + }, LOG_CONTEXTS.JENKINS); + res.status(500).json({ + success: false, + message, + details: { + error: message, + timestamp: new Date().toISOString(), + suggestions: [ + '检查请求头中的认证信息', + '验证 IP 地址是否在白名单中', + '确保请求格式正确' + ] + } + }); + } +}); + +/** + * POST /api/jenkins/callback/manual-sync/:runId + * 手动同步执行状态 - 用于修复卡住的运行记录 + * 从数据库查询当前状态并允许手动更新 + * 通过 IP 白名单验证 + */ +router.post('/callback/manual-sync/:runId', [ + ipWhitelistMiddleware.verify, + rateLimitMiddleware.limit +], async (req: Request, res: Response) => { + try { + const runId = parseInt(req.params.runId); + const syncBody = (req.body ?? {}) as Record; + const status = syncBody['status']; + const passedCases = syncBody['passedCases']; + const failedCases = syncBody['failedCases']; + const skippedCases = syncBody['skippedCases']; + const durationMs = syncBody['durationMs']; + const results = syncBody['results']; + const force = typeof syncBody['force'] === 'boolean' ? syncBody['force'] : false; + const normalizedManualResults = Array.isArray(results) ? normalizeCallbackResults(results) : []; + + if (isNaN(runId) || runId <= 0) { + return res.status(400).json({ + success: false, + message: 'Invalid runId parameter. Must be a positive integer.' + }); + } + + logger.info(`Starting manual sync for execution`, { + runId, + status, + passedCases, + failedCases, + skippedCases, + durationMs, + resultsCount: normalizedManualResults.length, + force, + timestamp: new Date().toISOString() + }, LOG_CONTEXTS.JENKINS); + + // 查询现有运行记录 + const execution = await executionService.getBatchExecution(runId); + + if (!execution.execution) { + return res.status(404).json({ + success: false, + message: `Execution not found: runId=${runId}` + }); + } + + const executionData = execution.execution as unknown as Record; + const currentStatus = executionData['status']; + + // 检查是否允许更新 + if (!force && ['success', 'failed', 'cancelled'].includes(currentStatus as string)) { + return res.status(400).json({ + success: false, + message: `Execution is already completed with status: ${currentStatus}. Use force=true to override.`, + current: { + id: runId, + status: currentStatus, + totalCases: executionData['total_cases'], + passedCases: executionData['passed_cases'], + failedCases: executionData['failed_cases'], + skippedCases: executionData['skipped_cases'], + updatedAt: executionData['updated_at'] ?? executionData['created_at'] + } + }); + } + + // 必须提供新状态 + if (!status) { + return res.status(400).json({ + success: false, + message: 'status field is required for manual sync' + }); + } + + // 执行更新 + const startTime = Date.now(); + + await executionService.completeBatchExecution(runId, { + status: status as 'success' | 'failed' | 'cancelled', + passedCases: typeof passedCases === 'number' ? passedCases : 0, + failedCases: typeof failedCases === 'number' ? failedCases : 0, + skippedCases: typeof skippedCases === 'number' ? skippedCases : 0, + durationMs: typeof durationMs === 'number' ? durationMs : 0, + results: normalizedManualResults, + }); + + const processingTime = Date.now() - startTime; + + logger.info(`Successfully completed manual sync for execution`, { + runId, + processingTimeMs: processingTime, + timestamp: new Date().toISOString(), + }, LOG_CONTEXTS.JENKINS); + + // 查询更新后的数据 + const updated = await executionService.getBatchExecution(runId); + + const updatedData = updated.execution as unknown as Record; + + res.json({ + success: true, + message: 'Manual sync completed successfully', + previous: { + id: runId, + status: currentStatus, + totalCases: executionData['total_cases'], + passedCases: executionData['passed_cases'], + failedCases: executionData['failed_cases'], + skippedCases: executionData['skipped_cases'] + }, + updated: { + id: runId, + status: updatedData['status'], + totalCases: updatedData['total_cases'], + passedCases: updatedData['passed_cases'], + failedCases: updatedData['failed_cases'], + skippedCases: updatedData['skipped_cases'], + endTime: updatedData['end_time'], + durationMs: updatedData['duration_ms'] + }, + timing: { + processingTimeMs: processingTime, + timestamp: new Date().toISOString() + } + }); + + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + const errorDetails = error instanceof Error ? error.stack : undefined; + + logger.error(`Failed to complete manual sync for execution`, { + event: LOG_EVENTS.JENKINS_MANUAL_SYNC_FAILED, + runId: req.params.runId, + error: message, + stack: errorDetails, + timestamp: new Date().toISOString() + }, LOG_CONTEXTS.JENKINS); + + res.status(500).json({ + success: false, + message: `Manual sync failed: ${message}`, + details: { + error: message, + timestamp: new Date().toISOString(), + suggestions: [ + '检查 runId 是否存在于数据库', + '确保传入的状态值有效(success、failed、aborted)', + '查看后端日志获取详细错误信息', + '如果执行已完成,使用 force=true 强制更新' + ] + } + }); + } +}); + +/** + * POST /api/jenkins/callback/diagnose + * 诊断回调连接问题 - 通过 IP 白名单验证以保护系统信息 + * + * 安全建议:建议添加管理员权限验证 + * TODO: 添加 requireAuth 和 requireRole('admin') 中间件以增强安全性 + */ +router.post('/callback/diagnose', + generalAuthRateLimiter, + optionalAuth, // 添加可选认证,获取用户信息 + rateLimitMiddleware.limit, + ipWhitelistMiddleware.verify, + async (req: Request, res: Response) => { + // 检查用户权限(如果已认证) + if (req.user && process.env.NODE_ENV === 'production') { + // 在生产环境中,建议检查用户是否为管理员 + // if (req.user.role !== 'admin') { + // return res.status(403).json({ + // success: false, + // message: 'Access denied. Admin privileges required.' + // }); + // } + logger.info('Diagnostic request from authenticated user', { + userId: req.user.id, + userEmail: req.user.email, + }, LOG_CONTEXTS.JENKINS); + } + try { + const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; + const timestamp = new Date().toISOString(); + + logger.debug(`Received callback diagnostic request`, { + clientIP, + timestamp, + headers: Object.keys(req.headers).filter(k => k.toLowerCase().includes('auth') || k.toLowerCase().includes('jenkins')) + }, LOG_CONTEXTS.JENKINS); + + // 分析回调配置 + const envConfig = { + jenkins_url: !!process.env.JENKINS_URL, + jenkins_user: !!process.env.JENKINS_USER, + jenkins_token: !!process.env.JENKINS_TOKEN, + jenkins_allowed_ips: !!process.env.JENKINS_ALLOWED_IPS, + }; + const diagnostics: { + timestamp: string; + clientIP: string; + environmentVariablesConfigured: typeof envConfig; + requestHeaders: Record; + suggestions: string[]; + nextSteps?: string[]; + } = { + timestamp, + clientIP, + environmentVariablesConfigured: envConfig, + requestHeaders: { + hasContentType: !!req.headers['content-type'], + }, + suggestions: [], + }; + + // 分析问题并给出建议 + if (!diagnostics.environmentVariablesConfigured.jenkins_token) { + diagnostics.suggestions.push('⚠️ 未配置 JENKINS_TOKEN,Jenkins API 集成可能无法正常工作'); + } + if (!diagnostics.environmentVariablesConfigured.jenkins_allowed_ips) { + diagnostics.suggestions.push('⚠️ 未配置 JENKINS_ALLOWED_IPS,将允许所有 IP 访问回调接口'); + } + + if (diagnostics.suggestions.length === 0) { + diagnostics.suggestions.push('✅ 所有必需的环境变量已配置'); + diagnostics.suggestions.push('✅ 回调接口已就绪'); + } + + // 提供配置步骤 + diagnostics.nextSteps = [ + '1️⃣ 配置 JENKINS_ALLOWED_IPS 以限制回调源 IP(推荐)', + '2️⃣ 配置 JENKINS_URL、JENKINS_USER、JENKINS_TOKEN 用于 API 集成', + '3️⃣ 使用 curl 测试回调:', + ' curl -X POST http://localhost:3000/api/jenkins/callback/test \\', + ' -H "Content-Type: application/json" \\', + ' -d \'{"testMessage": "hello"}\'', + '4️⃣ 如果收到成功响应,可以开始集成 Jenkins', + '📚 详细文档:docs/JENKINS_CONFIG_GUIDE.md' + ]; + + res.json({ + success: true, + data: diagnostics, + message: 'Diagnostic report generated' + }); + + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`Callback diagnostic failed`, { + event: LOG_EVENTS.JENKINS_DIAGNOSE_FAILED, + error: message, + }, LOG_CONTEXTS.JENKINS); + res.status(500).json({ + success: false, + message: `Diagnostic failed: ${message}` + }); + } +}); + +/** + * GET /api/jenkins/health + * Jenkins 连接健康检查 - 包括详细的诊断信息 + */ +} diff --git a/server/routes/jenkinsDiagnosticRoutes.ts b/server/routes/jenkinsDiagnosticRoutes.ts new file mode 100644 index 0000000..9bf7413 --- /dev/null +++ b/server/routes/jenkinsDiagnosticRoutes.ts @@ -0,0 +1,704 @@ +import { Router, Request, Response } from 'express'; +import { In } from 'typeorm'; +import { executionService, type Auto_TestRunResultsInput } from '../services/ExecutionService'; +import { jenkinsService } from '../services/JenkinsService'; +import { jenkinsStatusService } from '../services/JenkinsStatusService'; +import { taskSchedulerService } from '../services/TaskSchedulerService'; +import { callbackQueue, type CallbackPayload } from '../services/CallbackQueue'; +import { ipWhitelistMiddleware, rateLimitMiddleware } from '../middleware/JenkinsAuthMiddleware'; +import { requestValidator } from '../middleware/RequestValidator'; +import { generalAuthRateLimiter } from '../middleware/authRateLimiter'; +import { optionalAuth } from '../middleware/auth'; +import logger from '../utils/logger'; +import { buildJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnostics'; +import { persistJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnosticArtifact'; +import { validateScriptPathsInTestRepo } from '../utils/testRepoScriptPathValidator'; +import { LOG_CONTEXTS, LOG_EVENTS, createTimer } from '../config/logging'; +import { AppDataSource, query, queryOne } from '../config/database'; +import { TestCase } from '../entities/TestCase'; +import { hybridSyncService } from '../services/HybridSyncService'; +import { executionMonitorService } from '../services/ExecutionMonitorService'; +import { + CALLBACK_TERMINAL_STATUSES, + deriveCallbackTerminalStatus, + normalizeCallbackTerminalStatus, +} from '../services/ExecutionService/callbackStatus'; +import { + DEFAULT_JENKINS_URL, + DEFAULT_JENKINS_USER, + HEALTH_CHECK_TIMEOUT_MS, + buildCallbackUrl, + normalizeCallbackResults, + preflightExecutableScriptPaths, + recordTriggerFailure, + resolveExecutionBusinessError, + resolveScriptPaths, + runJenkinsTriggerPrecheck, + sanitizeErrorMessage, + scheduleCallbackFallbackSync, + warnIfCallbackUrlIsLocal, +} from './jenkinsRouteSupport'; + +export function registerJenkinsDiagnosticRoutes(router: Router): void { +router.get('/health', generalAuthRateLimiter, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + const startTime = Date.now(); + + try { + logger.info(`Starting Jenkins health check...`, {}, LOG_CONTEXTS.JENKINS); + + // 测试 Jenkins 连接 + // 生产环境强制要求配置 Jenkins 环境变量 + if (process.env.NODE_ENV === 'production') { + if (!process.env.JENKINS_URL || !process.env.JENKINS_USER || !process.env.JENKINS_TOKEN) { + return res.status(500).json({ + success: false, + message: 'Jenkins configuration is missing in production environment', + data: { + connected: false, + details: { + issues: [ + '❌ 生产环境缺少必需的 Jenkins 配置', + !process.env.JENKINS_URL ? '❌ JENKINS_URL 未配置' : '', + !process.env.JENKINS_USER ? '❌ JENKINS_USER 未配置' : '', + !process.env.JENKINS_TOKEN ? '❌ JENKINS_TOKEN 未配置' : '', + ].filter(Boolean), + recommendations: [ + '请在环境变量中配置 JENKINS_URL', + '请在环境变量中配置 JENKINS_USER', + '请在环境变量中配置 JENKINS_TOKEN', + ], + }, + }, + }); + } + } + + const jenkinsUrl = process.env.JENKINS_URL || DEFAULT_JENKINS_URL; + const jenkinsUser = process.env.JENKINS_USER || DEFAULT_JENKINS_USER; + const jenkinsToken = process.env.JENKINS_TOKEN || ''; + + // 健康检查数据 + const healthCheckData: { + timestamp: string; + duration: number; + checks: Record; + diagnostics: Record; + issues: string[]; + recommendations: string[]; + } = { + timestamp: new Date().toISOString(), + duration: 0, + checks: { + connectionTest: { success: false, duration: 0 }, + authenticationTest: { success: false, duration: 0 }, + apiResponseTest: { success: false, duration: 0 }, + targetJobInspection: { success: false, duration: 0 }, + }, + diagnostics: { + configPresent: { + url: !!jenkinsUrl, + user: !!jenkinsUser, + token: !!jenkinsToken, + } + }, + issues: [] as string[], + recommendations: [] as string[], + }; + + // 1. 测试基础连接 + logger.debug(`Testing connection to Jenkins`, { + jenkinsUrl, + }, LOG_CONTEXTS.JENKINS); + const connStartTime = Date.now(); + + // 构建 API URL(处理 URL 尾部斜杠) + let apiUrl = jenkinsUrl; + if (!apiUrl.endsWith('/')) { + apiUrl += '/'; + } + apiUrl += 'api/json'; + + logger.debug(`Final API URL for health check`, { + apiUrl, + }, LOG_CONTEXTS.JENKINS); + + const credentials = Buffer.from(`${jenkinsUser}:${jenkinsToken}`).toString('base64'); + + // 设置超时 + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS); + + try { + const response = await fetch(apiUrl, { + method: 'GET', + headers: { + 'Authorization': `Basic ${credentials}`, + 'Content-Type': 'application/json', + }, + signal: controller.signal, + }); + + clearTimeout(timeoutId); + healthCheckData.checks.connectionTest.duration = Date.now() - connStartTime; + healthCheckData.checks.connectionTest.success = response.ok; + healthCheckData.diagnostics.connectionStatus = response.status; + healthCheckData.diagnostics.statusText = response.statusText; + + logger.debug(`Jenkins health check response received`, { + status: response.status, + statusText: response.statusText, + duration: healthCheckData.checks.connectionTest.duration, + }, LOG_CONTEXTS.JENKINS); + + if (response.ok) { + const data = await response.json() as Record; + healthCheckData.checks.authenticationTest.success = true; + healthCheckData.checks.apiResponseTest.success = true; + let triggerReady = false; + + const targetJobStart = Date.now(); + try { + const targetJobInspection = await jenkinsService.inspectConfiguredApiJob(); + healthCheckData.checks.targetJobInspection.duration = Date.now() - targetJobStart; + healthCheckData.checks.targetJobInspection.success = Boolean(targetJobInspection?.triggerReady); + healthCheckData.diagnostics.targetJobInspection = targetJobInspection; + triggerReady = Boolean(targetJobInspection?.triggerReady); + + if (targetJobInspection) { + healthCheckData.issues.push(...targetJobInspection.issues); + healthCheckData.recommendations.push(...targetJobInspection.recommendations); + } + } catch (inspectionError) { + healthCheckData.checks.targetJobInspection.duration = Date.now() - targetJobStart; + healthCheckData.diagnostics.targetJobInspectionError = + inspectionError instanceof Error ? inspectionError.message : String(inspectionError); + healthCheckData.issues.push('❌ 无法读取目标 Jenkins Job 的实时配置'); + healthCheckData.recommendations.push('检查 Jenkins Job 权限,确保当前账号具备读取任务配置的权限。'); + } + + healthCheckData.duration = Date.now() - startTime; + + res.json({ + success: true, + data: { + connected: true, + triggerReady, + jenkinsUrl, + version: typeof data['version'] === 'string' ? data['version'] : 'unknown', + timestamp: new Date().toISOString(), + details: healthCheckData, + }, + message: triggerReady + ? 'Jenkins is healthy' + : 'Jenkins is reachable, but the target job needs configuration fixes before the platform can trigger it' + }); + } else if (response.status === 401 || response.status === 403) { + healthCheckData.issues.push('❌ 认证失败:API Token 或用户名可能不正确'); + healthCheckData.recommendations.push('检查 JENKINS_USER 和 JENKINS_TOKEN 环境变量'); + + res.status(response.status).json({ + success: false, + data: { + connected: false, + status: response.status, + statusText: response.statusText, + details: healthCheckData, + }, + message: 'Jenkins service authentication failed. Please check configuration.' + }); + } else { + healthCheckData.issues.push(`❌ Jenkins 返回错误状态: ${response.status} ${response.statusText}`); + healthCheckData.recommendations.push('检查 Jenkins 服务是否正常运行'); + healthCheckData.recommendations.push('检查 JENKINS_URL 是否正确'); + + res.status(response.status).json({ + success: false, + data: { + connected: false, + status: response.status, + statusText: response.statusText, + details: healthCheckData, + }, + message: `Jenkins returned ${response.status}: ${response.statusText}` + }); + } + } catch (fetchError) { + clearTimeout(timeoutId); + + const fetchErrorMsg = fetchError instanceof Error ? fetchError.message : String(fetchError); + healthCheckData.checks.connectionTest.duration = Date.now() - connStartTime; + + if (fetchErrorMsg.includes('ECONNREFUSED')) { + healthCheckData.issues.push('❌ 连接被拒绝:Jenkins 服务可能未运行'); + healthCheckData.recommendations.push('确保 Jenkins 服务已启动'); + } else if (fetchErrorMsg.includes('ENOTFOUND')) { + healthCheckData.issues.push('❌ DNS 解析失败:无法解析 Jenkins 域名'); + healthCheckData.recommendations.push('检查 JENKINS_URL 中的域名是否正确'); + healthCheckData.recommendations.push('检查网络连接和 DNS 配置'); + } else if (fetchErrorMsg.includes('Aborted')) { + healthCheckData.issues.push('❌ 请求超时:Jenkins 响应时间过长(> 10秒)'); + healthCheckData.recommendations.push('检查 Jenkins 服务状态和网络连接'); + healthCheckData.recommendations.push('考虑增加超时时间'); + } else { + healthCheckData.issues.push(`❌ 网络错误:${fetchErrorMsg}`); + } + + throw fetchError; + } + } catch (error: unknown) { + const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_HEALTH'); + + res.status(500).json({ + success: false, + data: { + connected: false, + error: sanitizedMessage, + details: { + timestamp: new Date().toISOString(), + duration: Date.now() - startTime, + issues: [ + '❌ 无法连接到 Jenkins', + '请检查Jenkins服务状态和网络连接' + ], + recommendations: [ + '检查 Jenkins 服务是否运行', + '检查网络连接', + '验证 Jenkins URL 配置', + '查看应用日志获取详细错误信息' + ] + }, + stack: process.env.NODE_ENV === 'development' ? (error instanceof Error ? error.stack : undefined) : undefined + }, + message: `Failed to connect to Jenkins: ${sanitizedMessage}` + }); + } +}); + +/** + * GET /api/jenkins/diagnose + * 诊断执行问题 - 通过 IP 白名单验证以保护系统信息 + */ +router.get('/diagnose', + generalAuthRateLimiter, + rateLimitMiddleware.limit, + ipWhitelistMiddleware.verify, + async (req: Request, res: Response) => { + try { + const runId = parseInt(req.query.runId as string); + + if (isNaN(runId) || runId <= 0) { + return res.status(400).json({ + success: false, + message: 'Invalid runId parameter. Must be a positive integer.' + }); + } + + logger.info(`Starting execution diagnosis`, { + runId, + }, LOG_CONTEXTS.JENKINS); + + // 获取执行批次信息 + const batch = await executionService.getBatchExecution(runId); + const execution = batch.execution; + + // 计算执行时长 + const startTime = execution.startTime ? new Date(execution.startTime).getTime() : null; + const currentTime = Date.now(); + const executionDuration = startTime ? currentTime - startTime : 0; + + // 检查Jenkins连接状态 + let jenkinsConnectivity: any = null; + if (execution.jenkinsJob && execution.jenkinsBuildId) { + try { + const buildStatus = await jenkinsStatusService.getBuildStatus( + execution.jenkinsJob as string, + execution.jenkinsBuildId as string + ); + jenkinsConnectivity = { + canConnect: !!buildStatus, + buildStatus: buildStatus ? { + building: buildStatus.building, + result: buildStatus.result, + duration: buildStatus.duration, + url: buildStatus.url + } : null + }; + } catch (error) { + jenkinsConnectivity = { + canConnect: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } + } + + // 收集诊断信息 + const diagnostics = { + executionId: execution.id, + status: execution.status, + jenkinsJob: execution.jenkinsJob, + jenkinsBuildId: execution.jenkinsBuildId, + jenkinsUrl: execution.jenkinsUrl, + startTime: execution.startTime, + createdAt: execution.createdAt, + totalCases: execution.totalCases, + passedCases: execution.passedCases, + failedCases: execution.failedCases, + skippedCases: execution.skippedCases, + executionDuration, + + // 诊断信息 + diagnostics: { + jenkinsInfoMissing: !execution.jenkinsJob || !execution.jenkinsBuildId || !execution.jenkinsUrl, + startTimeMissing: !execution.startTime, + stillPending: execution.status === 'pending', + stillRunning: execution.status === 'running', + noTestResults: execution.passedCases === 0 && execution.failedCases === 0 && execution.skippedCases === 0, + longRunning: executionDuration > 5 * 60 * 1000, // 超过5分钟 + veryLongRunning: executionDuration > 10 * 60 * 1000, // 超过10分钟 + jenkinsConnectivity, + + // 时间分析 + timeAnalysis: { + executionAge: executionDuration, + executionAgeMinutes: Math.round(executionDuration / 60000), + isOld: executionDuration > 30 * 60 * 1000, // 超过30分钟 + createdRecently: startTime && execution.createdAt ? (currentTime - new Date(execution.createdAt).getTime()) < 60 * 1000 : false + }, + + // 建议 + suggestions: [] as string[] + } + }; + + // 生成建议 + const sugg = diagnostics.diagnostics.suggestions; + + if (diagnostics.diagnostics.jenkinsInfoMissing) { + sugg.push('🚨 Jenkins 信息未被填充。这通常表示 Jenkins 触发失败。请检查后端日志查找错误信息。'); + } + + if (diagnostics.diagnostics.startTimeMissing) { + sugg.push('⏳ 执行开始时间为空。这表示 Jenkins 尚未开始构建。请等待几秒后重试。'); + } + + if (diagnostics.diagnostics.stillPending) { + if (diagnostics.diagnostics.timeAnalysis.executionAgeMinutes > 2) { + sugg.push('⚠️ 执行已处于 pending 状态超过2分钟,可能存在问题。建议手动同步状态。'); + } else { + sugg.push('⏳ 执行仍处于 pending 状态。这是正常的,系统正在等待 Jenkins 接收任务。'); + } + } + + if (diagnostics.diagnostics.stillRunning) { + if (diagnostics.diagnostics.veryLongRunning) { + sugg.push('🚨 执行已运行超过10分钟,可能卡住了。建议检查Jenkins构建状态或手动同步。'); + } else if (diagnostics.diagnostics.longRunning) { + sugg.push('⚠️ 执行已运行超过5分钟,请检查是否正常。可以尝试手动同步状态。'); + } + } + + if (diagnostics.diagnostics.noTestResults && !diagnostics.diagnostics.stillPending) { + sugg.push('❌ 测试结果为空。这可能表示 Jenkins 任务失败或回调未到达。请检查 Jenkins 的执行日志。'); + } + + // Jenkins连接性建议 + if (jenkinsConnectivity) { + if (!jenkinsConnectivity.canConnect) { + sugg.push('🔌 无法连接到Jenkins获取构建状态。请检查Jenkins服务器状态和网络连接。'); + } else if (jenkinsConnectivity.buildStatus) { + const buildStatus = jenkinsConnectivity.buildStatus; + if (!buildStatus.building && buildStatus.result) { + if (execution.status === 'running') { + sugg.push(`🔄 Jenkins显示构建已完成(${buildStatus.result}),但平台状态仍为running。建议立即手动同步。`); + } + } + } + } + + // 基于时间的建议 + if (diagnostics.diagnostics.timeAnalysis.isOld) { + sugg.push('🕐 执行时间过长(超过30分钟),建议检查或取消该执行。'); + } + + if (sugg.length === 0) { + sugg.push('✅ 执行状态良好,无明显问题。'); + } + + res.json({ + success: true, + data: diagnostics + }); + + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`Execution diagnosis failed`, { + event: LOG_EVENTS.JENKINS_DIAGNOSE_FAILED, + error: message, + }, LOG_CONTEXTS.JENKINS); + res.status(500).json({ + success: false, + message: `Diagnosis failed: ${message}` + }); + } +}); + +/** + * GET /api/jenkins/monitoring/stats + * 获取监控统计信息 + */ +router.get('/monitoring/stats', generalAuthRateLimiter, rateLimitMiddleware.limit, async (_req, res) => { + try { + logger.info(`Getting monitoring statistics...`, {}, LOG_CONTEXTS.JENKINS); + + // 获取混合同步服务的统计信息 + const syncStats = hybridSyncService.getMonitoringStats(); + + // 获取最近的执行统计 + const recentExecutions = await executionService.getRecentExecutions(50) as any[]; + const statusCounts = recentExecutions.reduce((acc: Record, exec: any) => { + acc[exec.status] = (acc[exec.status] || 0) + 1; + return acc; + }, {}); + + // 计算卡住的执行数量 + const stuckExecutions = recentExecutions.filter((exec: any) => { + if (!['running', 'pending'].includes(exec.status) || !exec.start_time) return false; + const duration = Date.now() - new Date(exec.start_time).getTime(); + return duration > 5 * 60 * 1000; // 超过5分钟 + }); + + const stats = { + timestamp: new Date().toISOString(), + syncService: syncStats, + executions: { + total: recentExecutions.length, + byStatus: statusCounts, + stuck: stuckExecutions.length, + stuckList: stuckExecutions.map((exec: any) => ({ + id: exec.id, + status: exec.status, + duration: Date.now() - new Date(exec.start_time).getTime(), + jenkins_job: exec.jenkins_job, + jenkins_build_id: exec.jenkins_build_id + })) + }, + health: { + totalIssues: syncStats.failed + syncStats.timeout + stuckExecutions.length, + hasIssues: (syncStats.failed + syncStats.timeout + stuckExecutions.length) > 0 + } + }; + + res.json({ + success: true, + data: stats + }); + + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`Failed to get monitoring statistics`, { + event: LOG_EVENTS.JENKINS_MONITORING_STATS_FAILED, + error: message, + }, LOG_CONTEXTS.JENKINS); + res.status(500).json({ + success: false, + message: `Failed to get monitoring stats: ${message}` + }); + } +}); + +/** + * POST /api/jenkins/monitoring/fix-stuck + * 修复卡住的执行 + */ +router.post('/monitoring/fix-stuck', generalAuthRateLimiter, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + try { + const fixBody = (req.body ?? {}) as Record; + const timeoutMinutes = typeof fixBody['timeoutMinutes'] === 'number' ? fixBody['timeoutMinutes'] : 5; + const dryRun = typeof fixBody['dryRun'] === 'boolean' ? fixBody['dryRun'] : false; + + logger.info(`${dryRun ? 'Simulating' : 'Starting'} fix for stuck executions`, { + timeoutMinutes, + dryRun, + }, LOG_CONTEXTS.JENKINS); + + if (dryRun) { + // 只查询,不修复 + const timeoutMs = timeoutMinutes * 60 * 1000; + const timeoutThreshold = new Date(Date.now() - timeoutMs); + + const stuckExecutions = await query(` + SELECT id, status, jenkins_job, jenkins_build_id, jenkins_url, + start_time, TIMESTAMPDIFF(MINUTE, start_time, NOW()) as duration_minutes + FROM Auto_TestRun + WHERE status IN ('pending', 'running') + AND start_time < ? + ORDER BY start_time ASC + LIMIT 20 + `, [timeoutThreshold]) as any[]; + + res.json({ + success: true, + data: { + dryRun: true, + wouldFix: stuckExecutions.length, + executions: stuckExecutions + } + }); + } else { + // 实际修复 + const timeoutMs = timeoutMinutes * 60 * 1000; + const result = await executionService.checkAndHandleTimeouts(timeoutMs); + + res.json({ + success: true, + data: { + dryRun: false, + checked: result.checked, + updated: result.updated, + timedOut: result.timedOut, + message: `Fixed ${result.updated} executions, marked ${result.timedOut} as timed out` + } + }); + } + + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error(`Failed to fix stuck executions`, { + event: LOG_EVENTS.JENKINS_FIX_STUCK_FAILED, + error: message, + }, LOG_CONTEXTS.JENKINS); + res.status(500).json({ + success: false, + message: `Failed to fix stuck executions: ${message}` + }); + } +}); + +/** + * GET /api/jenkins/monitor/status + * Get execution monitor service status and statistics + */ +router.get('/monitor/status', generalAuthRateLimiter, rateLimitMiddleware.limit, async (_req: Request, res: Response) => { + try { + const status = executionMonitorService.getStatus(); + const stats = executionMonitorService.getStats(); + + logger.debug('Monitor status requested', { + isRunning: status.isRunning, + cyclesRun: stats.cyclesRun, + }, LOG_CONTEXTS.MONITOR); + + res.json({ + success: true, + data: { + status: status.isRunning ? 'running' : 'stopped', + isRunning: status.isRunning, + config: status.config, + stats: { + cyclesRun: stats.cyclesRun, + totalExecutionsChecked: stats.totalExecutionsChecked, + totalExecutionsUpdated: stats.totalExecutionsUpdated, + totalCompilationFailures: stats.totalCompilationFailures, + totalErrors: stats.totalErrors, + lastCycleTime: stats.lastCycleTime, + lastCycleDuration: stats.lastCycleDuration, + isProcessing: stats.isProcessing, + }, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + logger.error('Failed to get monitor status', { + event: LOG_EVENTS.JENKINS_MONITOR_STATUS_FAILED, + error: message, + }, LOG_CONTEXTS.MONITOR); + res.status(500).json({ + success: false, + message: `Failed to get monitor status: ${message}`, + }); + } +}); + +/** + * GET /api/jenkins/metrics + * 获取 Jenkins 集成相关的所有监控指标(P2-C) + * + * 聚合指标: + * - rateLimit: 429 次数、每分钟 429 速率、活跃 IP 数 + * - callbackQueue: 队列深度、总入队/处理/失败数、平均排队时长、重试分布 + * - jenkinsQueue: queueId 轮询总次数、成功解析数、超时/取消数、平均/最大等待时长 + * - process: 内存使用、进程运行时长 + * + * 访问控制:需要认证(通过 optionalAuth 获取用户信息,如未认证则仅返回部分指标) + */ +router.get('/metrics', [generalAuthRateLimiter, optionalAuth], (_req: Request, res: Response) => { + try { + const rateLimitMetrics = rateLimitMiddleware.getMetrics(); + const queueMetrics = callbackQueue.getMetrics(); + const jenkinsQueueMetrics = jenkinsService.getQueueMetrics(); + const memUsage = process.memoryUsage(); + + res.json({ + success: true, + timestamp: new Date().toISOString(), + data: { + /** + * 速率限制指标 + */ + rateLimit: { + total429Count: rateLimitMetrics.total429Count, + rate429PerMinute: rateLimitMetrics.rate429PerMinute, + activeIPs: rateLimitMetrics.activeIPs, + }, + + /** + * 回调队列指标(P2-B) + */ + callbackQueue: { + queueDepth: queueMetrics.queueDepth, + workerBusy: queueMetrics.workerBusy, + totalEnqueued: queueMetrics.totalEnqueued, + totalProcessed: queueMetrics.totalProcessed, + totalFailed: queueMetrics.totalFailed, + avgWaitMs: queueMetrics.avgWaitMs, + maxWaitMs: queueMetrics.maxWaitMs, + retryDistribution: queueMetrics.retryDistribution, + // 最近 20 条排队时长样本(用于画趋势图) + recentWaitSamples: queueMetrics.waitTimeSamples.slice(-20), + }, + + /** + * Jenkins 构建队列指标(P2-A) + */ + jenkinsQueue: { + totalPolls: jenkinsQueueMetrics.totalPolls, + resolvedCount: jenkinsQueueMetrics.resolvedCount, + timeoutCount: jenkinsQueueMetrics.timeoutCount, + avgWaitMs: jenkinsQueueMetrics.avgWaitMs, + maxWaitMs: jenkinsQueueMetrics.maxWaitMs, + resolutionRate: jenkinsQueueMetrics.totalPolls > 0 + ? Math.round((jenkinsQueueMetrics.resolvedCount / jenkinsQueueMetrics.totalPolls) * 100) + : 0, + // 最近 20 条 Jenkins 队列等待时长样本 + recentWaitSamples: jenkinsQueueMetrics.waitTimeSamples.slice(-20), + }, + + /** + * 进程级指标 + */ + process: { + uptimeSeconds: Math.floor(process.uptime()), + memoryMB: { + rss: Math.round(memUsage.rss / 1024 / 1024), + heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024), + heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024), + }, + }, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, message }); + } +}); +} diff --git a/server/routes/jenkinsExecutionRoutes.ts b/server/routes/jenkinsExecutionRoutes.ts new file mode 100644 index 0000000..5382ade --- /dev/null +++ b/server/routes/jenkinsExecutionRoutes.ts @@ -0,0 +1,803 @@ +import { Router, Request, Response } from 'express'; +import { In } from 'typeorm'; +import { executionService, type Auto_TestRunResultsInput } from '../services/ExecutionService'; +import { jenkinsService } from '../services/JenkinsService'; +import { jenkinsStatusService } from '../services/JenkinsStatusService'; +import { taskSchedulerService } from '../services/TaskSchedulerService'; +import { callbackQueue, type CallbackPayload } from '../services/CallbackQueue'; +import { ipWhitelistMiddleware, rateLimitMiddleware } from '../middleware/JenkinsAuthMiddleware'; +import { requestValidator } from '../middleware/RequestValidator'; +import { generalAuthRateLimiter } from '../middleware/authRateLimiter'; +import { optionalAuth } from '../middleware/auth'; +import logger from '../utils/logger'; +import { buildJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnostics'; +import { persistJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnosticArtifact'; +import { validateScriptPathsInTestRepo } from '../utils/testRepoScriptPathValidator'; +import { LOG_CONTEXTS, LOG_EVENTS, createTimer } from '../config/logging'; +import { AppDataSource, query, queryOne } from '../config/database'; +import { TestCase } from '../entities/TestCase'; +import { hybridSyncService } from '../services/HybridSyncService'; +import { executionMonitorService } from '../services/ExecutionMonitorService'; +import { + CALLBACK_TERMINAL_STATUSES, + deriveCallbackTerminalStatus, + normalizeCallbackTerminalStatus, +} from '../services/ExecutionService/callbackStatus'; +import { + buildCallbackUrl, + normalizeCallbackResults, + preflightExecutableScriptPaths, + recordTriggerFailure, + resolveExecutionBusinessError, + resolveScriptPaths, + runJenkinsTriggerPrecheck, + sanitizeErrorMessage, + scheduleCallbackFallbackSync, + warnIfCallbackUrlIsLocal, +} from './jenkinsRouteSupport'; + +export function registerJenkinsExecutionRoutes(router: Router): void { +router.post('/trigger', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + try { + const triggerBody = (req.body ?? {}) as Record; + let caseIds = triggerBody['caseIds']; + const projectId = typeof triggerBody['projectId'] === 'number' ? triggerBody['projectId'] : 1; + // 优先使用认证用户 ID,回退到请求体中的 triggeredBy,最后才用默认值 1(系统管理员) + const triggeredBy = req.user?.id ?? (typeof triggerBody['triggeredBy'] === 'number' ? triggerBody['triggeredBy'] : 1); + const jenkinsJobName = typeof triggerBody['jenkinsJobName'] === 'string' ? triggerBody['jenkinsJobName'] : undefined; + const taskId = typeof triggerBody['taskId'] === 'number' ? triggerBody['taskId'] : undefined; + let taskName: string | undefined; + + // 如果传入了 taskId,从数据库查找任务信息 + if (taskId !== undefined) { + const task = await queryOne<{ id: number; name: string; case_ids: string; project_id: number }>( + 'SELECT id, name, case_ids, project_id FROM Auto_TestCaseTasks WHERE id = ?', + [taskId] + ); + + if (!task) { + return res.status(404).json({ + success: false, + message: `Task with id ${taskId} not found` + }); + } + + taskName = task.name; + + // 如果没有直接传入 caseIds,从任务中解析 + if (!caseIds || !Array.isArray(caseIds) || caseIds.length === 0) { + try { + const parsedCaseIds = JSON.parse(task.case_ids); + if (!Array.isArray(parsedCaseIds) || parsedCaseIds.length === 0) { + logger.warn('Task has empty or invalid case_ids', { + taskId, + case_ids: task.case_ids, + }, LOG_CONTEXTS.JENKINS); + return res.status(400).json({ + success: false, + message: `Task ${taskId} has no valid case_ids configured` + }); + } + caseIds = parsedCaseIds as number[]; + } catch (err) { + logger.error('Failed to parse task case_ids', { + event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, + taskId, + case_ids: task.case_ids, + error: err instanceof Error ? err.message : String(err), + }, LOG_CONTEXTS.JENKINS); + return res.status(500).json({ + success: false, + message: 'Failed to parse task configuration. Invalid JSON format in case_ids field.' + }); + } + } + } + + if (!caseIds || !Array.isArray(caseIds) || caseIds.length === 0) { + return res.status(400).json({ + success: false, + message: 'caseIds is required and must be a non-empty array (or provide a valid taskId with case_ids)' + }); + } + + // 创建运行记录 + const execution = await executionService.triggerTestExecution({ + caseIds: caseIds as number[], + projectId, + triggeredBy, + triggerType: 'jenkins', + jenkinsJob: jenkinsJobName, + taskId, + taskName, + }); + + res.json({ + success: true, + data: { + runId: execution.runId, + totalCases: execution.totalCases, + status: 'pending', + jenkinsJobName: jenkinsJobName || null, + message: 'Execution created. Waiting for Jenkins to start.' + } + }); + } catch (error: unknown) { + const businessError = resolveExecutionBusinessError(error); + if (businessError) { + return res.status(businessError.statusCode).json({ + success: false, + message: businessError.message, + details: businessError.details, + }); + } + + const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_TRIGGER'); + res.status(500).json({ success: false, message: sanitizedMessage }); + } +}); + +/** + * POST /api/jenkins/run-case + * 触发单个用例执行 + * + * 异步队列模式: + * 1. 立即创建执行记录(status=pending) + * 2. 立即返回 runId 给前端(不阻塞) + * 3. 后台通过 enqueueDirectJob 等待并发槽位,槽位可用后再触发 Jenkins + */ +router.post('/run-case', [ + generalAuthRateLimiter, + optionalAuth, + rateLimitMiddleware.limit, + requestValidator.validateSingleExecution +], async (req: Request, res: Response) => { + const timer = createTimer(); + const { caseId, projectId } = req.body; + const triggeredBy: number = req.user?.id ?? (typeof req.body.triggeredBy === 'number' ? req.body.triggeredBy : 1); + const slotLabel = `case:${caseId}`; + + try { + logger.info('Starting single case execution (async queue mode)', { + caseId, + projectId, + triggeredBy, + }, LOG_CONTEXTS.JENKINS); + + const precheck = await runJenkinsTriggerPrecheck('run-case'); + if (!precheck.ok) { + return res.status(503).json({ + success: false, + message: `Jenkins 当前不可用,请稍后重试(${precheck.reason})`, + details: { + reason: precheck.reason, + source: 'run-case-precheck', + retryable: true, + }, + }); + } + + // ── Step 1: 立即创建执行记录(状态 pending)────────────── + const scriptPathPreflight = await preflightExecutableScriptPaths([caseId]); + if (!scriptPathPreflight.ok) { + return res.status(scriptPathPreflight.statusCode).json({ + success: false, + message: scriptPathPreflight.message, + details: scriptPathPreflight.details, + }); + } + + const preflightScriptPaths = scriptPathPreflight.scriptPaths; + + const execution = await executionService.triggerTestExecution({ + caseIds: [caseId], + projectId, + triggeredBy, + triggerType: 'manual', + }); + + logger.info('Execution record created, returning runId immediately', { + runId: execution.runId, + executionId: execution.executionId, + }, LOG_CONTEXTS.JENKINS); + + // ── Step 2: 立即返回 runId,不等待 Jenkins ─────────────── + const duration = timer(); + res.json({ + success: true, + data: { + runId: execution.runId, + status: 'queued', + }, + message: '任务已加入执行队列', + _concurrency: { + slotsUsed: taskSchedulerService.getStatus().running.length, + slotsLimit: taskSchedulerService.getStatus().concurrencyLimit, + directQueued: taskSchedulerService.getStatus().directQueueDepth, + }, + }); + + // ── Step 3: 后台异步等待槽位 + 触发 Jenkins ────────────── + const capturedRunId = execution.runId; + + try { + taskSchedulerService.enqueueDirectJob(slotLabel, async (placeholderRunId: number) => { + // 槽位获取后,用真实 runId 替换占位槽位 + taskSchedulerService.registerDirectSlot(capturedRunId, slotLabel, placeholderRunId); + + try { + // 解析脚本路径 + const callbackUrl = buildCallbackUrl(); + + // 触发 Jenkins + const triggerResult = await jenkinsService.triggerBatchJob( + capturedRunId, + [caseId], + preflightScriptPaths, + callbackUrl, + async (buildNumber: number, buildUrl: string, queueWaitMs: number) => { + const buildId = String(buildNumber); + logger.debug('[dev-10] Build resolved via queueId poll, updating Jenkins info', { + runId: capturedRunId, + buildId, + buildUrl, + queueWaitMs, + }, LOG_CONTEXTS.JENKINS); + await executionService.updateBatchJenkinsInfo(capturedRunId, { buildId, buildUrl }); + scheduleCallbackFallbackSync(capturedRunId, 'run-case'); + }, + async (reason: 'cancelled' | 'timeout') => { + logger.warn('[dev-11] Jenkins queue cancelled/timeout, marking execution as aborted', { + runId: capturedRunId, + reason, + }, LOG_CONTEXTS.JENKINS); + try { + await executionService.markExecutionAborted(capturedRunId, `Jenkins build ${reason}`); + } catch (err) { + logger.warn('[dev-11] Failed to mark execution as aborted', { + runId: capturedRunId, + error: err instanceof Error ? err.message : String(err), + }, LOG_CONTEXTS.JENKINS); + } + taskSchedulerService.releaseSlotByRunId(capturedRunId); + } + ); + + if (!triggerResult.success) { + // Jenkins 触发失败,立即释放槽位 + taskSchedulerService.releaseSlotByRunId(capturedRunId); + // 将执行状态标记为失败 + try { + await recordTriggerFailure(capturedRunId, [caseId], preflightScriptPaths, callbackUrl, 'run-case', triggerResult); + } catch { /* ignore */ } + logger.warn('[run-case] Jenkins trigger failed (async), slot released', { + runId: capturedRunId, + message: triggerResult.message, + }, LOG_CONTEXTS.JENKINS); + } else { + logger.info('[run-case] Jenkins trigger success (async)', { + runId: capturedRunId, + queueId: triggerResult.queueId, + }, LOG_CONTEXTS.JENKINS); + } + } catch (jenkinsErr) { + // Jenkins 执行异常,释放槽位并标记失败 + taskSchedulerService.releaseSlotByRunId(capturedRunId); + try { + await executionService.markExecutionAborted(capturedRunId, `Jenkins error: ${jenkinsErr instanceof Error ? jenkinsErr.message : String(jenkinsErr)}`); + } catch { /* ignore */ } + logger.errorLog(jenkinsErr, '[run-case] Async Jenkins trigger error', { runId: capturedRunId, caseId }); + } + }); + } catch (queueErr) { + // 仅当队列已满才会到这里(enqueueDirectJob 同步抛出) + // runId 已返回给前端,将执行状态标记为失败 + const queueErrMsg = queueErr instanceof Error ? queueErr.message : '并发队列已满'; + logger.warn('[run-case] Queue full, marking execution as aborted', { + runId: capturedRunId, + message: queueErrMsg, + }, LOG_CONTEXTS.JENKINS); + try { + await executionService.markExecutionAborted(capturedRunId, queueErrMsg); + } catch { /* ignore */ } + } + + } catch (error: unknown) { + const duration = timer(); + logger.errorLog(error, 'Single case execution failed (creating record)', { + caseId, + projectId, + durationMs: duration, + }); + + const businessError = resolveExecutionBusinessError(error); + if (businessError) { + return res.status(businessError.statusCode).json({ + success: false, + message: businessError.message, + details: businessError.details, + }); + } + + const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_RUN_CASE'); + res.status(500).json({ success: false, message: sanitizedMessage }); + } +}); + +/** + * POST /api/jenkins/run-batch + * 触发批量用例执行 + * + * 异步队列模式: + * 1. 立即创建执行记录(status=pending) + * 2. 立即返回 runId 给前端(不阻塞) + * 3. 后台通过 enqueueDirectJob 等待并发槽位,槽位可用后再触发 Jenkins + */ +router.post('/run-batch', [ + generalAuthRateLimiter, + optionalAuth, + rateLimitMiddleware.limit, + requestValidator.validateBatchExecution +], async (req: Request, res: Response) => { + const timer = createTimer(); + const { caseIds, projectId } = req.body; + const triggeredBy: number = req.user?.id ?? (typeof req.body.triggeredBy === 'number' ? req.body.triggeredBy : 1); + // label 展示前几个 caseId,避免过长 + const labelIds = (caseIds as number[]).slice(0, 3).join(',') + (caseIds.length > 3 ? `…(${caseIds.length})` : ''); + const slotLabel = `batch:${labelIds}`; + + try { + logger.info('Starting batch case execution (async queue mode)', { + caseCount: caseIds.length, + caseIds, + projectId, + triggeredBy, + }, LOG_CONTEXTS.JENKINS); + + const precheck = await runJenkinsTriggerPrecheck('run-batch'); + if (!precheck.ok) { + return res.status(503).json({ + success: false, + message: `Jenkins 当前不可用,请稍后重试(${precheck.reason})`, + details: { + reason: precheck.reason, + source: 'run-batch-precheck', + retryable: true, + }, + }); + } + + // ── Step 1: 立即创建执行记录(状态 pending)────────────── + const scriptPathPreflight = await preflightExecutableScriptPaths(caseIds); + if (!scriptPathPreflight.ok) { + return res.status(scriptPathPreflight.statusCode).json({ + success: false, + message: scriptPathPreflight.message, + details: scriptPathPreflight.details, + }); + } + + const preflightScriptPaths = scriptPathPreflight.scriptPaths; + + const execution = await executionService.triggerTestExecution({ + caseIds, + projectId, + triggeredBy, + triggerType: 'manual', + }); + + logger.info('Batch execution record created, returning runId immediately', { + runId: execution.runId, + executionId: execution.executionId, + totalCases: execution.totalCases, + }, LOG_CONTEXTS.JENKINS); + + // ── Step 2: 立即返回 runId,不等待 Jenkins ─────────────── + const duration = timer(); + res.json({ + success: true, + data: { + runId: execution.runId, + totalCases: execution.totalCases, + status: 'queued', + }, + message: '任务已加入执行队列', + _concurrency: { + slotsUsed: taskSchedulerService.getStatus().running.length, + slotsLimit: taskSchedulerService.getStatus().concurrencyLimit, + directQueued: taskSchedulerService.getStatus().directQueueDepth, + }, + }); + + // ── Step 3: 后台异步等待槽位 + 触发 Jenkins ────────────── + const capturedRunId = execution.runId; + + try { + taskSchedulerService.enqueueDirectJob(slotLabel, async (placeholderRunId: number) => { + // 槽位获取后,用真实 runId 替换占位槽位 + taskSchedulerService.registerDirectSlot(capturedRunId, slotLabel, placeholderRunId); + + try { + // 解析脚本路径 + const callbackUrl = buildCallbackUrl(); + + // 触发 Jenkins + const triggerResult = await jenkinsService.triggerBatchJob( + capturedRunId, + caseIds, + preflightScriptPaths, + callbackUrl, + async (buildNumber: number, buildUrl: string, queueWaitMs: number) => { + const buildId = String(buildNumber); + logger.debug('[dev-10] Build resolved via queueId poll, updating batch Jenkins info', { + runId: capturedRunId, + buildId, + buildUrl, + queueWaitMs, + }, LOG_CONTEXTS.JENKINS); + await executionService.updateBatchJenkinsInfo(capturedRunId, { buildId, buildUrl }); + scheduleCallbackFallbackSync(capturedRunId, 'run-batch'); + }, + async (reason: 'cancelled' | 'timeout') => { + logger.warn('[dev-11] Batch Jenkins queue cancelled/timeout, marking execution as aborted', { + runId: capturedRunId, + reason, + }, LOG_CONTEXTS.JENKINS); + try { + await executionService.markExecutionAborted(capturedRunId, `Jenkins build ${reason}`); + } catch (err) { + logger.warn('[dev-11] Failed to mark batch execution as aborted', { + runId: capturedRunId, + error: err instanceof Error ? err.message : String(err), + }, LOG_CONTEXTS.JENKINS); + } + taskSchedulerService.releaseSlotByRunId(capturedRunId); + } + ); + + if (!triggerResult.success) { + taskSchedulerService.releaseSlotByRunId(capturedRunId); + try { + await recordTriggerFailure(capturedRunId, caseIds, preflightScriptPaths, callbackUrl, 'run-batch', triggerResult); + } catch { /* ignore */ } + logger.warn('[run-batch] Jenkins trigger failed (async), slot released', { + runId: capturedRunId, + message: triggerResult.message, + }, LOG_CONTEXTS.JENKINS); + } else { + logger.info('[run-batch] Jenkins trigger success (async)', { + runId: capturedRunId, + queueId: triggerResult.queueId, + }, LOG_CONTEXTS.JENKINS); + } + } catch (jenkinsErr) { + taskSchedulerService.releaseSlotByRunId(capturedRunId); + try { + await executionService.markExecutionAborted(capturedRunId, `Jenkins error: ${jenkinsErr instanceof Error ? jenkinsErr.message : String(jenkinsErr)}`); + } catch { /* ignore */ } + logger.errorLog(jenkinsErr, '[run-batch] Async Jenkins trigger error', { runId: capturedRunId, caseIds }); + } + }); + } catch (queueErr) { + const queueErrMsg = queueErr instanceof Error ? queueErr.message : '并发队列已满'; + logger.warn('[run-batch] Queue full, marking execution as aborted', { + runId: capturedRunId, + message: queueErrMsg, + }, LOG_CONTEXTS.JENKINS); + try { + await executionService.markExecutionAborted(capturedRunId, queueErrMsg); + } catch { /* ignore */ } + } + + } catch (error: unknown) { + const duration = timer(); + logger.errorLog(error, 'Batch case execution failed (creating record)', { + caseIds, + projectId, + durationMs: duration, + }); + + const businessError = resolveExecutionBusinessError(error); + if (businessError) { + return res.status(businessError.statusCode).json({ + success: false, + message: businessError.message, + details: businessError.details, + }); + } + + const sanitizedMessage = sanitizeErrorMessage(error, 'JENKINS_RUN_BATCH'); + res.status(500).json({ success: false, message: sanitizedMessage }); + } +}); + +/** + * GET /api/jenkins/tasks/:taskId/cases + * 获取任务关联的用例列表 + * + * Jenkins Job 可以调用此接口获取需要执行的用例信息 + */ +router.get('/tasks/:taskId/cases', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + try { + const taskId = parseInt(req.params.taskId); + if (isNaN(taskId) || taskId <= 0) { + return res.status(400).json({ + success: false, + message: 'Invalid taskId parameter. Must be a positive integer.' + }); + } + const cases = await executionService.getRunCases(taskId); + + res.json({ + success: true, + data: cases + }); + } catch (error: unknown) { + logger.errorLog(error, 'Failed to get task cases', { + event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, + taskId: req.params.taskId, + }); + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, message }); + } +}); + +/** + * GET /api/jenkins/status/:executionId + * 查询执行状态(预留接口) + * + * 用于查询 Jenkins Job 的执行状态 + */ +router.get('/status/:executionId', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + try { + const executionId = parseInt(req.params.executionId); + if (isNaN(executionId) || executionId <= 0) { + return res.status(400).json({ + success: false, + message: 'Invalid executionId parameter. Must be a positive integer.' + }); + } + const detail = await executionService.getExecutionDetail(executionId); + + if (!detail || !detail.execution) { + return res.status(404).json({ success: false, message: 'Execution not found' }); + } + + const execution = detail.execution as unknown as Record; + + res.json({ + success: true, + data: { + executionId, + status: execution['status'], + totalCases: execution['total_cases'], + passedCases: execution['passed_cases'], + failedCases: execution['failed_cases'], + skippedCases: execution['skipped_cases'], + startTime: execution['start_time'], + endTime: execution['end_time'], + duration: execution['duration'], + // Jenkins 相关字段(预留) + jenkinsStatus: null, + buildNumber: null, + consoleUrl: null + } + }); + } catch (error: unknown) { + logger.errorLog(error, 'Failed to get execution status', { + event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, + executionId: req.params.executionId, + }); + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, message }); + } +}); + +/** + * POST /api/jenkins/callback + * Jenkins 执行结果回调接口 + * 通过 IP 白名单验证,无需额外认证 + * 注意:此接口不使用 generalAuthRateLimiter,避免高并发回调时触发 429 + * 安全由 ipWhitelistMiddleware 白名单保护,并使用专用的 rateLimitMiddleware + */ +router.post('/callback', [ + ipWhitelistMiddleware.verify, + rateLimitMiddleware.limit, + requestValidator.validateCallback +], (req: Request, res: Response) => { + /** + * [P2-B] 快速 ACK 模式 + * 1. 仅做轻量校验和数据规范化(同步、无 I/O) + * 2. 将任务入队到 callbackQueue + * 3. 立即返回 202 Accepted,Jenkins 不会超时重试 + * 4. 后台 worker 异步消费队列,执行 completeBatchExecution + releaseSlot + */ + const receiveTimeMs = Date.now(); + const clientIP = req.ip || req.socket?.remoteAddress || 'unknown'; + + const { + runId, + status, + passedCases: reportedPassedCases = 0, + failedCases: reportedFailedCases = 0, + skippedCases: reportedSkippedCases = 0, + durationMs = 0, + results = [], + // 轻量化回调模式:仅发送 buildNumber + buildNumber, + } = req.body; + + // 判断是否为轻量化回调(有 buildNumber 但无 results) + const isLightweightCallback = !Array.isArray(results) || results.length === 0; + + const rawResults = Array.isArray(results) ? results : []; + const normalizedResults = normalizeCallbackResults(rawResults); + let passedCases = typeof reportedPassedCases === 'number' ? reportedPassedCases : 0; + let failedCases = typeof reportedFailedCases === 'number' ? reportedFailedCases : 0; + let skippedCases = typeof reportedSkippedCases === 'number' ? reportedSkippedCases : 0; + + // 从详细结果推导计数(与旧逻辑一致) + if (normalizedResults.length > 0) { + let derivedPassed = 0; + let derivedFailed = 0; + let derivedSkipped = 0; + + for (const result of normalizedResults) { + const caseStatus = String(result['status'] || '').toLowerCase(); + if (caseStatus === 'passed') derivedPassed++; + else if (caseStatus === 'failed' || caseStatus === 'error') derivedFailed++; + else derivedSkipped++; + } + + const totalReported = passedCases + failedCases + skippedCases; + const totalDerived = derivedPassed + derivedFailed + derivedSkipped; + const shouldUseDerived = totalReported === 0 + || totalReported !== normalizedResults.length + || totalReported !== totalDerived; + + if (shouldUseDerived) { + logger.warn('Callback summary mismatch, using derived counts', { + runId, + reported: { passedCases, failedCases, skippedCases, total: totalReported }, + derived: { passedCases: derivedPassed, failedCases: derivedFailed, skippedCases: derivedSkipped, total: totalDerived }, + resultsCount: normalizedResults.length, + }, LOG_CONTEXTS.JENKINS); + passedCases = derivedPassed; + failedCases = derivedFailed; + skippedCases = derivedSkipped; + } + } + + // 规范化状态值 + const normalizedReportedStatus = normalizeCallbackTerminalStatus(status); + + if (normalizedReportedStatus !== status) { + logger.warn('Invalid callback status, treating as failed', { + runId, + providedStatus: status, + validStatuses: CALLBACK_TERMINAL_STATUSES, + }, LOG_CONTEXTS.JENKINS); + } + + const hasCallbackSummary = (passedCases + failedCases + skippedCases) > 0; + const normalizedStatus = hasCallbackSummary + ? deriveCallbackTerminalStatus({ + reportedStatus: normalizedReportedStatus, + passedCases, + failedCases, + skippedCases, + }) + : normalizedReportedStatus; + + logger.info('Jenkins callback received, enqueuing for async processing', { + runId, + status: normalizedStatus, + passedCases, + failedCases, + skippedCases, + durationMs, + resultsCount: normalizedResults.length, + clientIP, + userAgent: req.get('User-Agent'), + receiveTimeMs, + isLightweightCallback, + buildNumber, + }, LOG_CONTEXTS.JENKINS); + + // 入队(非阻塞) + const enqueued = callbackQueue.enqueue({ + runId, + status: normalizedStatus, + passedCases, + failedCases, + skippedCases, + durationMs, + results: normalizedResults, + // 轻量化回调参数 + buildNumber: isLightweightCallback ? buildNumber : undefined, + needsServerParsing: isLightweightCallback, + }); + + if (!enqueued) { + // 队列已满:返回 429 让 Jenkins 稍后重试 + rateLimitMiddleware.increment429Count(); + logger.error('Callback queue full, returning 429', { + event: LOG_EVENTS.JENKINS_CALLBACK_QUEUE_FULL, + runId, + queueMetrics: callbackQueue.getMetrics(), + }, LOG_CONTEXTS.JENKINS); + return res.status(429).json({ + success: false, + message: 'Callback queue is full. Please retry later.', + retryAfter: 5, + }); + } + + // 快速 ACK(202 Accepted:已接受,正在异步处理) + const ackTimeMs = Date.now() - receiveTimeMs; + return res.status(202).json({ + success: true, + message: 'Callback accepted for async processing', + ackTimeMs, + }); +}); + +/** + * GET /api/jenkins/batch/:runId + * 获取执行批次详情 + */ +router.get('/batch/:runId', generalAuthRateLimiter, optionalAuth, rateLimitMiddleware.limit, async (req: Request, res: Response) => { + try { + const runId = parseInt(req.params.runId); + if (isNaN(runId) || runId <= 0) { + return res.status(400).json({ + success: false, + message: 'Invalid runId parameter. Must be a positive integer.' + }); + } + const batch = await executionService.getBatchExecution(runId); + + const e = batch.execution; + + // 将 TypeORM entity 的 camelCase 字段映射为 snake_case,与前端 TestRunRecord 接口对齐 + res.json({ + success: true, + data: { + id: e.id, + project_id: e.projectId ?? null, + project_name: null, + status: e.status, + trigger_type: e.triggerType, + trigger_by: e.triggerBy, + trigger_by_name: e.triggerByName ?? null, + jenkins_job: e.jenkinsJob ?? null, + jenkins_build_id: e.jenkinsBuildId ?? null, + jenkins_url: e.jenkinsUrl ?? null, + total_cases: e.totalCases, + passed_cases: e.passedCases, + failed_cases: e.failedCases, + skipped_cases: e.skippedCases, + duration_ms: e.durationMs, + start_time: e.startTime ?? null, + end_time: e.endTime ?? null, + created_at: e.createdAt, + } + }); + } catch (error: unknown) { + logger.errorLog(error, 'Failed to get batch execution', { + event: LOG_EVENTS.JENKINS_CALLBACK_FAILED, + runId: req.params.runId, + }); + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, message }); + } +}); + +/** + * POST /api/jenkins/callback/test + * 测试回调连接 - 支持传入真实数据进行测试处理 + * 可选参数: runId, status, passedCases, failedCases, skippedCases, durationMs, results + * 如果提供了 runId,则会真实处理回调数据;否则仅测试连接 + * 通过 IP 白名单验证 + */ +} diff --git a/server/routes/jenkinsRouteSupport.ts b/server/routes/jenkinsRouteSupport.ts new file mode 100644 index 0000000..e9439d8 --- /dev/null +++ b/server/routes/jenkinsRouteSupport.ts @@ -0,0 +1,712 @@ +import { Router, Request, Response } from 'express'; +import { In } from 'typeorm'; +import { executionService, type Auto_TestRunResultsInput } from '../services/ExecutionService'; +import { jenkinsService } from '../services/JenkinsService'; +import { jenkinsStatusService } from '../services/JenkinsStatusService'; +import { taskSchedulerService } from '../services/TaskSchedulerService'; +import { callbackQueue, type CallbackPayload } from '../services/CallbackQueue'; +import { ipWhitelistMiddleware, rateLimitMiddleware } from '../middleware/JenkinsAuthMiddleware'; +import { requestValidator } from '../middleware/RequestValidator'; +import { generalAuthRateLimiter } from '../middleware/authRateLimiter'; +import { optionalAuth } from '../middleware/auth'; +import logger from '../utils/logger'; +import { buildJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnostics'; +import { persistJenkinsTriggerFailureDiagnostic } from '../utils/jenkinsTriggerDiagnosticArtifact'; +import { validateScriptPathsInTestRepo } from '../utils/testRepoScriptPathValidator'; +import { LOG_CONTEXTS, LOG_EVENTS, createTimer } from '../config/logging'; +import { AppDataSource, query, queryOne } from '../config/database'; +import { TestCase } from '../entities/TestCase'; +import { hybridSyncService } from '../services/HybridSyncService'; +import { executionMonitorService } from '../services/ExecutionMonitorService'; +import { + CALLBACK_TERMINAL_STATUSES, + deriveCallbackTerminalStatus, + normalizeCallbackTerminalStatus, +} from '../services/ExecutionService/callbackStatus'; + +// ──────────────────────────────────────────────────────────────────────────── +// 常量定义 +// ──────────────────────────────────────────────────────────────────────────── + +/** 回调兜底同步默认延迟(毫秒) */ +const DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS = 45_000; +/** 回调兜底同步最小延迟(毫秒) */ +const MIN_CALLBACK_FALLBACK_SYNC_DELAY_MS = 10_000; +/** Jenkins 健康检查超时(毫秒) */ +export const HEALTH_CHECK_TIMEOUT_MS = 5_000; +/** Jenkins 健康检查默认 URL */ +export const DEFAULT_JENKINS_URL = 'http://jenkins.wiac.xyz'; +/** Jenkins 健康检查默认用户 */ +export const DEFAULT_JENKINS_USER = 'root'; +/** 触发前 Jenkins 预检查默认超时(毫秒) */ +const DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS = 8_000; +/** 触发前 Jenkins 预检查超时(毫秒) */ +const TRIGGER_PRECHECK_TIMEOUT_MS = Math.max( + 1_000, + Number.parseInt( + process.env.JENKINS_TRIGGER_PRECHECK_TIMEOUT_MS ?? String(DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS), + 10 + ) || DEFAULT_TRIGGER_PRECHECK_TIMEOUT_MS +); +/** 触发前 Jenkins 预检查重试次数(总尝试次数 = 1 + retries) */ +const TRIGGER_PRECHECK_RETRIES = Math.max( + 0, + Math.min(3, Number.parseInt(process.env.JENKINS_TRIGGER_PRECHECK_RETRIES ?? '1', 10) || 1) +); +/** 触发前 Jenkins 预检查重试间隔(毫秒) */ +const TRIGGER_PRECHECK_RETRY_DELAY_MS = Math.max( + 200, + Number.parseInt(process.env.JENKINS_TRIGGER_PRECHECK_RETRY_DELAY_MS ?? '600', 10) || 600 +); +/** 是否启用触发前 Jenkins 预检查 */ +// 注:Jenkins 预检查默认禁用(当 Jenkins 网络不稳定时) +// 设置 JENKINS_TRIGGER_PRECHECK_ENABLED=true 以启用 +// 启用后,当 Jenkins 无法连接时,任务触发请求会被拒绝 (503 Service Unavailable) +const TRIGGER_PRECHECK_ENABLED = (process.env.JENKINS_TRIGGER_PRECHECK_ENABLED ?? 'false') !== 'false'; + +/** + * [P2-B] 注册 CallbackQueue 消费者 + * 将 completeBatchExecution + releaseSlot 的整个处理流程注入队列 worker + * 在路由模块初始化时立即注册,确保消费者在第一个请求到来之前已就绪 + * + * 支持两种模式: + * 1. 全量回调:Jenkins 解析结果后发送完整数据(兼容旧模式) + * 2. 轻量化回调:Jenkins 仅发送 buildNumber,服务端主动解析结果 + */ +callbackQueue.register(async (payload: CallbackPayload) => { + let shouldReleaseSlot = false; + try { + let finalPayload = payload; + + // ─── 轻量化回调:服务端主动解析结果 ───────────────────────────── + if (payload.needsServerParsing) { + logger.info('[CallbackQueue] Lightweight callback detected, parsing results from Jenkins', { + runId: payload.runId, + buildNumber: payload.buildNumber, + }, LOG_CONTEXTS.JENKINS); + + try { + // 从数据库获取执行记录,提取 jenkinsJob 名称 + const batch = await executionService.getBatchExecution(payload.runId); + const execution = batch?.execution; + const buildNumber = payload.buildNumber ?? execution?.jenkinsBuildId; + + if (execution?.jenkinsJob && buildNumber) { + const testResults = await jenkinsStatusService.parseBuildResults( + execution.jenkinsJob as string, + String(buildNumber) + ); + + if (testResults) { + const reportedStatus = normalizeCallbackTerminalStatus(payload.status); + // 使用解析结果覆盖 payload + finalPayload = { + runId: payload.runId, + status: deriveCallbackTerminalStatus({ + reportedStatus, + passedCases: testResults.passedCases, + failedCases: testResults.failedCases, + skippedCases: testResults.skippedCases, + }), + passedCases: testResults.passedCases, + failedCases: testResults.failedCases, + skippedCases: testResults.skippedCases, + durationMs: testResults.duration || payload.durationMs, + results: testResults.results.map(r => ({ + caseId: r.caseId, + caseName: r.caseName, + status: r.status, + duration: r.duration, + errorMessage: r.errorMessage, + stackTrace: r.stackTrace, + })), + }; + + logger.info('[CallbackQueue] Successfully parsed results from Jenkins', { + runId: payload.runId, + buildNumber, + status: finalPayload.status, + passedCases: finalPayload.passedCases, + failedCases: finalPayload.failedCases, + skippedCases: finalPayload.skippedCases, + }, LOG_CONTEXTS.JENKINS); + } else { + // 解析失败,降级为使用构建状态 + logger.warn('[CallbackQueue] Failed to parse results from Jenkins, falling back to build status', { + runId: payload.runId, + buildNumber, + fallbackStatus: payload.status, + }, LOG_CONTEXTS.JENKINS); + } + } else { + logger.warn('[CallbackQueue] No jenkins job/build number found for execution, cannot parse results', { + runId: payload.runId, + buildNumber, + jenkinsJob: execution?.jenkinsJob, + jenkinsBuildId: execution?.jenkinsBuildId, + }, LOG_CONTEXTS.JENKINS); + } + } catch (parseError) { + logger.error('Failed to parse build results in lightweight callback', { + event: LOG_EVENTS.JENKINS_CALLBACK_PARSE_FAILED, + runId: payload.runId, + buildNumber: payload.buildNumber, + error: parseError instanceof Error ? parseError.message : String(parseError), + }, LOG_CONTEXTS.JENKINS); + // 解析异常,继续使用原始 payload(降级处理) + } + } + + const normalizedReportedStatus = normalizeCallbackTerminalStatus(finalPayload.status); + const hasCallbackSummary = (finalPayload.passedCases + finalPayload.failedCases + finalPayload.skippedCases) > 0; + finalPayload = { + ...finalPayload, + status: hasCallbackSummary + ? deriveCallbackTerminalStatus({ + reportedStatus: normalizedReportedStatus, + passedCases: finalPayload.passedCases, + failedCases: finalPayload.failedCases, + skippedCases: finalPayload.skippedCases, + }) + : normalizedReportedStatus, + }; + + await executionService.completeBatchExecution(finalPayload.runId, { + status: finalPayload.status, + passedCases: finalPayload.passedCases, + failedCases: finalPayload.failedCases, + skippedCases: finalPayload.skippedCases, + durationMs: finalPayload.durationMs, + results: finalPayload.results as Parameters[1]['results'], + }); + // 只在成功完成后标记需要释放槽位 + shouldReleaseSlot = true; + } catch (error) { + // 如果 completeBatchExecution 失败,让 CallbackQueue 的重试机制处理 + // 不释放槽位,避免重复释放 + logger.warn('[CallbackQueue] completeBatchExecution failed, will retry', { + runId: payload.runId, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.EXECUTION); + throw error; // 重新抛出错误以触发重试 + } finally { + // 只在成功完成后释放并发槽位 + if (shouldReleaseSlot) { + taskSchedulerService.releaseSlotByRunId(payload.runId); + logger.debug('[CallbackQueue] Slot released after successful completion', { + runId: payload.runId, + }, LOG_CONTEXTS.EXECUTION); + } + } +}); + +const CALLBACK_FALLBACK_SYNC_DELAY_MS = Math.max( + MIN_CALLBACK_FALLBACK_SYNC_DELAY_MS, + Number.parseInt( + process.env.CALLBACK_FALLBACK_SYNC_DELAY_MS ?? String(DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS), + 10 + ) || DEFAULT_CALLBACK_FALLBACK_SYNC_DELAY_MS +); + +/** + * 当 Jenkins 回调丢失时,延迟触发一次兜底同步,避免状态长时间停留在 running/pending。 + */ +export function scheduleCallbackFallbackSync(runId: number, source: 'run-case' | 'run-batch'): void { + const timer = setTimeout(async () => { + try { + const detail = await executionService.getTestRunDetailRow(runId); + const currentStatus = String(detail.status ?? ''); + + if (!['pending', 'running'].includes(currentStatus)) { + logger.debug('[callback-fallback] execution already finalized, skipping sync', { + runId, + source, + currentStatus, + }, LOG_CONTEXTS.JENKINS); + return; + } + + const syncResult = await executionService.syncExecutionStatusFromJenkins(runId); + logger.info('[callback-fallback] fallback sync executed', { + runId, + source, + currentStatus, + delayMs: CALLBACK_FALLBACK_SYNC_DELAY_MS, + syncSuccess: syncResult.success, + syncUpdated: syncResult.updated, + jenkinsStatus: syncResult.jenkinsStatus, + message: syncResult.message, + }, LOG_CONTEXTS.JENKINS); + } catch (error) { + logger.warn('[callback-fallback] fallback sync failed', { + runId, + source, + delayMs: CALLBACK_FALLBACK_SYNC_DELAY_MS, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.JENKINS); + } + }, CALLBACK_FALLBACK_SYNC_DELAY_MS); + + timer.unref?.(); +} + +/** + * 构造 Jenkins 回调 URL:兼容配置基础地址或完整 callback 路径。 + */ +export function buildCallbackUrl(): string { + const configuredBase = (process.env.API_CALLBACK_URL ?? 'http://localhost:3000').trim(); + + // 兼容老配置:如果配置已包含 callback 路径,优先直接使用。 + const trimmed = configuredBase.replace(/\/+$/, ''); + if (trimmed.endsWith('/api/jenkins/callback')) { + warnIfCallbackUrlIsLocal(trimmed); + return trimmed; + } + + const callbackUrl = `${trimmed}/api/jenkins/callback`; + warnIfCallbackUrlIsLocal(callbackUrl); + return callbackUrl; +} + +export function warnIfCallbackUrlIsLocal(callbackUrl: string): void { + try { + const callbackHost = new URL(callbackUrl).hostname.toLowerCase(); + const jenkinsHost = new URL(process.env.JENKINS_URL || DEFAULT_JENKINS_URL).hostname.toLowerCase(); + const localHosts = new Set(['localhost', '127.0.0.1', '::1']); + + if (localHosts.has(callbackHost) && !localHosts.has(jenkinsHost)) { + logger.warn('Jenkins callback URL points to localhost while Jenkins is remote', { + event: 'JENKINS_CALLBACK_URL_LOCALHOST_FOR_REMOTE', + callbackUrl, + jenkinsHost, + suggestion: 'Set API_CALLBACK_URL to a URL that Jenkins can reach, otherwise callback may fail with 403 or never reach this service.', + }, LOG_CONTEXTS.JENKINS); + } + } catch (error) { + logger.warn('Failed to validate Jenkins callback URL', { + event: 'JENKINS_CALLBACK_URL_VALIDATE_FAILED', + callbackUrl, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.JENKINS); + } +} + +export async function recordTriggerFailure( + runId: number, + caseIds: number[], + scriptPaths: string[], + callbackUrl: string, + source: 'run-case' | 'run-batch', + triggerResult: { message: string; errorCategory: 'none' | 'network' | 'auth_failed' | 'not_found' | 'bad_request' | 'rate_limited' | 'server_error' } +): Promise { + const config = jenkinsService.getConfigInfo(); + const persisted = await persistJenkinsTriggerFailureDiagnostic(triggerResult, { + runId, + source, + baseUrl: config?.baseUrl, + jobName: config?.jobs.api, + callbackUrl, + caseIds, + scriptPaths, + }).catch(async (error: unknown) => { + logger.warn('Failed to persist Jenkins trigger diagnostic artifact', { + runId, + error: error instanceof Error ? error.message : String(error), + }, LOG_CONTEXTS.JENKINS); + + return { + publicPath: undefined, + diagnostic: buildJenkinsTriggerFailureDiagnostic(triggerResult, { + baseUrl: config?.baseUrl, + jobName: config?.jobs.api, + callbackUrl, + caseIds, + scriptPaths, + }), + }; + }); + + await executionService.recordTriggerFailureDiagnostics({ + runId, + caseIds, + errorMessage: persisted.diagnostic.errorMessage, + errorStack: persisted.diagnostic.errorStack, + logPath: persisted.publicPath, + }); + await executionService.markExecutionAborted(runId, persisted.diagnostic.abortReason); +} + +/** + * 执行触发前的 Jenkins 连通性预检查。 + * 目标:在 Jenkins 不可用时快速失败,避免创建后立刻变成 aborted 的运行记录。 + */ +export async function runJenkinsTriggerPrecheck(source: 'run-case' | 'run-batch'): Promise<{ ok: true } | { ok: false; reason: string }> { + const configError = jenkinsService.getTriggerConfigurationError(); + if (configError) { + logger.warn('[trigger-precheck] Jenkins trigger configuration invalid', { + source, + reason: configError, + }, LOG_CONTEXTS.JENKINS); + return { ok: false, reason: configError }; + } + + if (!TRIGGER_PRECHECK_ENABLED) { + return { ok: true }; + } + + const maxAttempts = 1 + TRIGGER_PRECHECK_RETRIES; + const reasons: string[] = []; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + let timeoutId: NodeJS.Timeout | undefined; + + try { + const timeoutPromise = new Promise<{ connected: false; message: string }>((resolve) => { + timeoutId = setTimeout(() => { + resolve({ + connected: false, + message: `Jenkins precheck timeout after ${TRIGGER_PRECHECK_TIMEOUT_MS}ms`, + }); + }, TRIGGER_PRECHECK_TIMEOUT_MS); + timeoutId.unref?.(); + }); + + const checkResult = await Promise.race([ + jenkinsService.testConnection(), + timeoutPromise, + ]); + + if (checkResult.connected) { + if (attempt > 1) { + logger.info('[trigger-precheck] Jenkins precheck recovered after retry', { + source, + attempt, + maxAttempts, + }, LOG_CONTEXTS.JENKINS); + } + return { ok: true }; + } + + const reason = checkResult.message || 'Jenkins unavailable'; + reasons.push(reason); + + // 配置缺失属于确定性失败,无需重试 + if (reason.includes('not configured')) { + break; + } + + logger.warn('[trigger-precheck] Jenkins unavailable in attempt', { + source, + attempt, + maxAttempts, + reason, + timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, + }, LOG_CONTEXTS.JENKINS); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + reasons.push(reason); + logger.warn('[trigger-precheck] Jenkins precheck attempt failed with exception', { + source, + attempt, + maxAttempts, + reason, + timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, + }, LOG_CONTEXTS.JENKINS); + } finally { + if (timeoutId) clearTimeout(timeoutId); + } + + if (attempt < maxAttempts) { + await new Promise((resolve) => { + const timer = setTimeout(() => resolve(), TRIGGER_PRECHECK_RETRY_DELAY_MS); + timer.unref?.(); + }); + } + } + + const reason = reasons[reasons.length - 1] || 'Jenkins unavailable'; + logger.warn('[trigger-precheck] Jenkins unavailable, rejecting trigger request after retries', { + source, + maxAttempts, + reason, + reasons, + timeoutMs: TRIGGER_PRECHECK_TIMEOUT_MS, + retryDelayMs: TRIGGER_PRECHECK_RETRY_DELAY_MS, + }, LOG_CONTEXTS.JENKINS); + + return { ok: false, reason }; +} + +/** + * 解析并去重脚本路径 + */ +export async function resolveScriptPaths(caseIds: number[]): Promise<{ scriptPaths: string[]; missingCaseIds: number[] }> { + if (!Array.isArray(caseIds) || caseIds.length === 0) { + return { scriptPaths: [], missingCaseIds: [] }; + } + + const cases = await AppDataSource.getRepository(TestCase).find({ + where: { + id: In(caseIds), + enabled: true, + }, + select: ['id', 'scriptPath'], + }); + + const scriptPathCaseIds = new Set(); + const normalizedPaths = new Set(); + + for (const item of cases) { + const path = item.scriptPath?.trim(); + if (path) { + scriptPathCaseIds.add(item.id); + normalizedPaths.add(path); + } + } + + const missingCaseIds = caseIds.filter(id => !scriptPathCaseIds.has(id)); + + return { + scriptPaths: Array.from(normalizedPaths), + missingCaseIds, + }; +} + +export async function preflightExecutableScriptPaths(caseIds: number[]): Promise< + | { ok: true; scriptPaths: string[] } + | { + ok: false; + statusCode: number; + message: string; + details: { + reason: 'missing_script_path' | 'script_path_not_found_in_repo'; + caseIds?: number[]; + missingPaths?: string[]; + }; + } +> { + const { scriptPaths, missingCaseIds } = await resolveScriptPaths(caseIds); + + if (missingCaseIds.length > 0) { + return { + ok: false, + statusCode: 400, + message: missingCaseIds.length === 1 + ? `测试用例 ${missingCaseIds[0]} 未配置 script_path,请先同步或修正后再执行` + : `存在未配置 script_path 的测试用例,请先同步或修正后再执行:${missingCaseIds.join(', ')}`, + details: { + reason: 'missing_script_path', + caseIds: missingCaseIds, + }, + }; + } + + const testRepoConfig = jenkinsService.getTestRepoConfig(); + const missingPaths = testRepoConfig + ? (await validateScriptPathsInTestRepo({ + repoUrl: testRepoConfig.repoUrl, + branch: testRepoConfig.branch, + scriptPaths, + })).missingPaths + : []; + + if (missingPaths.length > 0) { + return { + ok: false, + statusCode: 400, + message: missingPaths.length === 1 + ? `测试仓库中不存在脚本路径:${missingPaths[0]}` + : `测试仓库中存在无效脚本路径,请先同步或修正后再执行:${missingPaths.join(', ')}`, + details: { + reason: 'script_path_not_found_in_repo', + missingPaths, + }, + }; + } + + return { ok: true, scriptPaths }; +} + +/** + * 净化错误消息,移除敏感信息以防止信息泄露 + * @param error 原始错误对象 + * @param context 错误上下文,用于日志记录 + * @returns 净化后的错误消息 + */ +export function sanitizeErrorMessage(error: unknown, context: string): string { + const originalMessage = error instanceof Error ? error.message : 'Unknown error'; + + // 记录详细错误信息到服务器日志 + logger.error(`${context} - Detailed error info`, { + event: LOG_EVENTS.JENKINS_TRIGGER_FAILED, + message: originalMessage, + stack: error instanceof Error ? error.stack : undefined, + timestamp: new Date().toISOString(), + context, + }, LOG_CONTEXTS.JENKINS); + + // 检查是否包含敏感信息关键词 + const sensitiveKeywords = [ + 'password', 'token', 'secret', 'key', 'credential', + ]; + + const lowerMessage = originalMessage.toLowerCase(); + const containsSensitiveInfo = sensitiveKeywords.some(keyword => + lowerMessage.includes(keyword.toLowerCase()) + ); + + if (containsSensitiveInfo) { + // 包含敏感信息时返回通用错误消息 + return 'An internal error occurred. Please contact support if the issue persists.'; + } + + // 生产环境返回简化但有意义的错误消息(移除路径、IP 等敏感信息) + if (process.env.NODE_ENV === 'production') { + return originalMessage + .replace(/\/[^\s]+/g, '[path]') // 替换文件路径 + .replace(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/g, '[ip]') // 替换 IP 地址 + .replace(/:\d+/g, ':[port]') // 替换端口号 + .replace(/localhost/gi, '[host]') // 替换 localhost + .replace(/127\.0\.0\.1/g, '[host]'); // 替换本地 IP + } + + // 开发环境返回原始消息 + return originalMessage; +} + +export function resolveExecutionBusinessError(error: unknown): { + statusCode: number; + message: string; + details: { reason: 'inactive_case' | 'inactive_cases'; caseIds: number[] }; +} | null { + const originalMessage = error instanceof Error ? error.message : String(error ?? ''); + const noActiveCasesMatch = originalMessage.match(/No active test cases found with IDs:\s*(.+)$/i); + if (!noActiveCasesMatch) return null; + + const caseIds = noActiveCasesMatch[1] + .split(',') + .map(part => Number.parseInt(part.trim(), 10)) + .filter(Number.isFinite); + + if (caseIds.length === 0) return null; + + if (caseIds.length === 1) { + return { + statusCode: 400, + message: `测试用例 ${caseIds[0]} 未启用,请先启用后再执行`, + details: { + reason: 'inactive_case', + caseIds, + }, + }; + } + + return { + statusCode: 400, + message: `存在未启用的测试用例,请先启用后再执行:${caseIds.join(', ')}`, + details: { + reason: 'inactive_cases', + caseIds, + }, + }; +} + +/** + * 规范化 Jenkins 回调中的 results 载荷,兼容 camelCase/snake_case 字段。 + * 目标:确保后续 completeBatchExecution 能稳定回写用例明细,避免残留占位 error。 + */ +export function normalizeCallbackResults(results: unknown[]): Auto_TestRunResultsInput[] { + const toNumber = (value: unknown): number | undefined => { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim() !== '') { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return undefined; + }; + + const toOptionalString = (value: unknown): string | undefined => { + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; + } + return undefined; + }; + + const normalizeStatus = (value: unknown): Auto_TestRunResultsInput['status'] => { + const rawStatus = String(value ?? '').trim().toLowerCase(); + if (rawStatus === 'passed' || rawStatus === 'success' || rawStatus === 'pass') return 'passed'; + if (rawStatus === 'failed' || rawStatus === 'fail') return 'failed'; + if (rawStatus === 'skipped' || rawStatus === 'skip') return 'skipped'; + + // 记录未知状态 + logger.warn('Unknown test result status, treating as error', { + rawStatus, + originalValue: value, + }, LOG_CONTEXTS.JENKINS); + return 'error'; + }; + + return results.flatMap((item): Auto_TestRunResultsInput[] => { + if (!item || typeof item !== 'object' || Array.isArray(item)) return []; + const row = item as Record; + + const caseIdRaw = toNumber(row['caseId'] ?? row['case_id']); + const caseName = toOptionalString(row['caseName'] ?? row['case_name']); + + // 允许 caseId=0 的结果通过(如 pytest 等框架不携带 caseId 的场景), + // 由 updateTestResult 的 caseName fallback 机制完成匹配。 + // 但若 caseId 和 caseName 同时缺失,则过滤掉(无法匹配任何占位符记录)。 + const hasValidCaseId = caseIdRaw && caseIdRaw > 0; + if (!hasValidCaseId && !caseName) { + logger.warn('Filtered out test result: missing both caseId and caseName', { + row, + caseId: caseIdRaw, + caseName, + }, LOG_CONTEXTS.JENKINS); + return []; + } + + if (!hasValidCaseId) { + logger.debug('Test result has no valid caseId, will use caseName fallback matching', { + caseId: caseIdRaw, + caseName, + }, LOG_CONTEXTS.JENKINS); + } + + const durationRaw = toNumber(row['duration'] ?? row['durationMs'] ?? row['duration_ms']); + const assertionsTotal = toNumber(row['assertionsTotal'] ?? row['assertions_total']); + const assertionsPassed = toNumber(row['assertionsPassed'] ?? row['assertions_passed']); + const startTime = row['startTime'] ?? row['start_time']; + const endTime = row['endTime'] ?? row['end_time']; + const responseDataRaw = row['responseData'] ?? row['response_data']; + + return [{ + caseId: caseIdRaw, + caseName: caseName || `case_${caseIdRaw}`, + status: normalizeStatus(row['status']), + duration: durationRaw !== undefined ? Math.max(0, durationRaw) : 0, + errorMessage: toOptionalString(row['errorMessage'] ?? row['error_message']), + stackTrace: toOptionalString(row['stackTrace'] ?? row['errorStack'] ?? row['error_stack']), + screenshotPath: toOptionalString(row['screenshotPath'] ?? row['screenshot_path']), + logPath: toOptionalString(row['logPath'] ?? row['log_path']), + assertionsTotal, + assertionsPassed, + responseData: typeof responseDataRaw === 'string' + ? responseDataRaw + : (responseDataRaw !== undefined ? JSON.stringify(responseDataRaw) : undefined), + startTime: typeof startTime === 'string' || typeof startTime === 'number' ? startTime : undefined, + endTime: typeof endTime === 'string' || typeof endTime === 'number' ? endTime : undefined, + }]; + }); +} + +/** + * POST /api/jenkins/trigger + * 触发 Jenkins Job 执行 + * + * 此接口创建运行记录并返回 executionId,供 Jenkins 后续回调使用 + * 支持两种模式: + * 1. 直接传入 caseIds 数组 + * 2. 传入 taskId,自动从数据库查找任务的 caseIds 和任务名称 + */ diff --git a/server/services/JenkinsService.ts b/server/services/JenkinsService.ts index 2f1f1eb..110ef11 100644 --- a/server/services/JenkinsService.ts +++ b/server/services/JenkinsService.ts @@ -2,222 +2,39 @@ import { isMisconfiguredTestRepoUrl, normalizeGitRemoteUrl } from '../utils/jenk import { normalizeConfiguredJenkinsBaseUrl } from '../utils/jenkinsUrl'; import logger from '../utils/logger'; import { getSecretOrEnv } from '../utils/secrets'; +import { + extractParameterNamesFromApiPayload, + extractXmlValue, + extractXmlValues, + isRecord, + normalizeBaseUrl, + runGitCommand, +} from './JenkinsServiceUtils'; +import type { + BuildCancelledCallback, + BuildResolvedCallback, + CaseType, + JenkinsConfig, + JenkinsCrumb, + JenkinsErrorCategory, + JenkinsJobInspection, + JenkinsQueueItem, + JenkinsQueueMetrics, + JenkinsTriggerHttpResult, + JenkinsTriggerResult, +} from './JenkinsServiceTypes'; +export { isJenkinsErrorRetryable } from './JenkinsServiceTypes'; +export type { + BuildCancelledCallback, + BuildResolvedCallback, + CaseType, + JenkinsConfig, + JenkinsErrorCategory, + JenkinsJobInspection, + JenkinsQueueMetrics, + JenkinsTriggerResult, +} from './JenkinsServiceTypes'; -/** - * Jenkins 配置接口 - */ -export interface JenkinsConfig { - baseUrl: string; - username: string; - token: string; - jobs: { - api: string; - ui: string; - performance: string; - }; - testRepoUrl?: string; - testRepoBranch: string; -} - -export interface JenkinsJobInspection { - jobName: string; - jobUrl: string; - jobClass?: string; - definitionClass?: string; - parameterized: boolean; - parameterNames: string[]; - triggerReady: boolean; - scmUrl?: string; - branchSpec?: string; - scriptPath?: string; - hasTimerTrigger: boolean; - timerSpec?: string; - issues: string[]; - recommendations: string[]; -} - -function runGitCommand(command: string): string | undefined { - try { - const { execSync } = require('child_process') as typeof import('child_process'); - const output = execSync(command, { - cwd: process.cwd(), - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - }).trim(); - return output || undefined; - } catch { - return undefined; - } -} - -function normalizeBaseUrl(baseUrl: string): string { - return baseUrl.trim().replace(/\/+$/, ''); -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; -} - -function decodeXmlEntities(value: string): string { - return value - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, '\'') - .replace(/&/g, '&'); -} - -function extractXmlValue(xml: string, pattern: RegExp): string | undefined { - const match = pattern.exec(xml); - if (!match?.[1]) { - return undefined; - } - - return decodeXmlEntities(match[1].trim()); -} - -function extractXmlValues(xml: string, pattern: RegExp): string[] { - return Array.from( - new Set( - Array.from(xml.matchAll(pattern)) - .map((match) => match[1]?.trim()) - .filter((value): value is string => Boolean(value)) - .map(decodeXmlEntities) - ) - ); -} - -function extractParameterNamesFromApiPayload(payload: Record): string[] { - const collected = new Set(); - - for (const key of ['actions', 'property']) { - const source = payload[key]; - if (!Array.isArray(source)) { - continue; - } - - for (const entry of source) { - if (!isRecord(entry)) { - continue; - } - - const definitions = entry.parameterDefinitions; - if (!Array.isArray(definitions)) { - continue; - } - - for (const definition of definitions) { - if (isRecord(definition) && typeof definition.name === 'string' && definition.name.trim()) { - collected.add(definition.name.trim()); - } - } - } - } - - return Array.from(collected); -} - -/** - * 错误分类:用于区分应重试的错误和不应重试的错误 - */ -export type JenkinsErrorCategory = - | 'none' // 无错误(成功) - | 'network' // 网络不可达(DNS失败、连接拒绝、超时等),可重试 - | 'auth_failed' // 认证失败(401/403),不应重试 - | 'not_found' // 资源不存在(Job不存在,404),不应重试 - | 'bad_request' // 参数错误(400),不应重试 - | 'rate_limited' // 被限流(429),可重试 - | 'server_error'; // 服务端错误(5xx),可重试 - -/** - * 判断某类错误是否应该触发重试 - */ -export function isJenkinsErrorRetryable(category: JenkinsErrorCategory): boolean { - return category === 'network' || category === 'rate_limited' || category === 'server_error'; -} - -/** - * Jenkins 触发结果 - */ -export interface JenkinsTriggerResult { - success: boolean; - queueId?: number; - buildUrl?: string; - buildNumber?: number; - message: string; - /** 错误分类,用于调度器判断是否需要重试 */ - errorCategory: JenkinsErrorCategory; -} - -/** - * Jenkins Queue Item(通过 queueId 轮询获取的结果) - */ -interface JenkinsQueueItem { - id: number; - why: string | null; - cancelled: boolean; - executable?: { - number: number; - url: string; - }; -} - -interface JenkinsCrumb { - field: string; - value: string; - fetchedAt: number; -} - -interface JenkinsTriggerHttpResult { - response: Response; - errorText?: string; -} - -/** - * [dev-10] queueId → buildNumber 解析完成后的回调类型 - * 由调用方注入,异步回调中更新数据库 - * @param buildNumber Jenkins 真实构建号 - * @param buildUrl Jenkins 构建 URL - * @param queueWaitMs 构建在 Jenkins 队列中等待的时长(毫秒) - */ -export type BuildResolvedCallback = (buildNumber: number, buildUrl: string, queueWaitMs: number) => Promise; - -/** - * [dev-11] Jenkins 队列取消/超时时的回调类型 - * 当 pollQueueForBuild 返回 null(构建被取消或等待超时)时调用 - * @param reason 取消原因:'cancelled'(Jenkins 队列项被取消)或 'timeout'(轮询超时) - */ -export type BuildCancelledCallback = (reason: 'cancelled' | 'timeout') => Promise; - -/** - * 用例类型 - */ -export type CaseType = 'api' | 'ui' | 'performance'; - -/** - * Jenkins Queue 指标(内存存储,进程重启后重置) - */ -export interface JenkinsQueueMetrics { - /** queueId 轮询总次数 */ - totalPolls: number; - /** 成功解析出 buildNumber 的次数 */ - resolvedCount: number; - /** 轮询超时/取消次数 */ - timeoutCount: number; - /** 所有成功解析的队列等待时长(ms)列表,保留最近 1000 条 */ - waitTimeSamples: number[]; - /** 队列等待时长总和(ms,仅成功解析) */ - totalWaitMs: number; - /** 平均队列等待时长(ms) */ - avgWaitMs: number; - /** 最大队列等待时长(ms) */ - maxWaitMs: number; -} - -/** - * Jenkins 服务类 - * 负责与 Jenkins 交互,触发 Job 执行 - */ export class JenkinsService { private config: JenkinsConfig; diff --git a/server/services/JenkinsServiceTypes.ts b/server/services/JenkinsServiceTypes.ts new file mode 100644 index 0000000..ad16082 --- /dev/null +++ b/server/services/JenkinsServiceTypes.ts @@ -0,0 +1,92 @@ +export interface JenkinsConfig { + baseUrl: string; + username: string; + token: string; + jobs: { + api: string; + ui: string; + performance: string; + }; + testRepoUrl?: string; + testRepoBranch: string; +} + +export interface JenkinsJobInspection { + jobName: string; + jobUrl: string; + jobClass?: string; + definitionClass?: string; + parameterized: boolean; + parameterNames: string[]; + triggerReady: boolean; + scmUrl?: string; + branchSpec?: string; + scriptPath?: string; + hasTimerTrigger: boolean; + timerSpec?: string; + issues: string[]; + recommendations: string[]; +} + +export type JenkinsErrorCategory = + | 'none' + | 'network' + | 'auth_failed' + | 'not_found' + | 'bad_request' + | 'rate_limited' + | 'server_error'; + +export function isJenkinsErrorRetryable(category: JenkinsErrorCategory): boolean { + return category === 'network' || category === 'rate_limited' || category === 'server_error'; +} + +export interface JenkinsTriggerResult { + success: boolean; + queueId?: number; + buildUrl?: string; + buildNumber?: number; + message: string; + errorCategory: JenkinsErrorCategory; +} + +export interface JenkinsQueueItem { + id: number; + why: string | null; + cancelled: boolean; + executable?: { + number: number; + url: string; + }; +} + +export interface JenkinsCrumb { + field: string; + value: string; + fetchedAt: number; +} + +export interface JenkinsTriggerHttpResult { + response: Response; + errorText?: string; +} + +export type BuildResolvedCallback = ( + buildNumber: number, + buildUrl: string, + queueWaitMs: number +) => Promise; + +export type BuildCancelledCallback = (reason: 'cancelled' | 'timeout') => Promise; + +export type CaseType = 'api' | 'ui' | 'performance'; + +export interface JenkinsQueueMetrics { + totalPolls: number; + resolvedCount: number; + timeoutCount: number; + waitTimeSamples: number[]; + totalWaitMs: number; + avgWaitMs: number; + maxWaitMs: number; +} diff --git a/server/services/JenkinsServiceUtils.ts b/server/services/JenkinsServiceUtils.ts new file mode 100644 index 0000000..bbf7db5 --- /dev/null +++ b/server/services/JenkinsServiceUtils.ts @@ -0,0 +1,80 @@ +export function runGitCommand(command: string): string | undefined { + try { + const { execSync } = require('child_process') as typeof import('child_process'); + const output = execSync(command, { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return output || undefined; + } catch { + return undefined; + } +} + +export function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.trim().replace(/\/+$/, ''); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function decodeXmlEntities(value: string): string { + return value + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, '\'') + .replace(/&/g, '&'); +} + +export function extractXmlValue(xml: string, pattern: RegExp): string | undefined { + const match = pattern.exec(xml); + if (!match?.[1]) { + return undefined; + } + + return decodeXmlEntities(match[1].trim()); +} + +export function extractXmlValues(xml: string, pattern: RegExp): string[] { + return Array.from( + new Set( + Array.from(xml.matchAll(pattern)) + .map((match) => match[1]?.trim()) + .filter((value): value is string => Boolean(value)) + .map(decodeXmlEntities) + ) + ); +} + +export function extractParameterNamesFromApiPayload(payload: Record): string[] { + const collected = new Set(); + + for (const key of ['actions', 'property']) { + const source = payload[key]; + if (!Array.isArray(source)) { + continue; + } + + for (const entry of source) { + if (!isRecord(entry)) { + continue; + } + + const definitions = entry.parameterDefinitions; + if (!Array.isArray(definitions)) { + continue; + } + + for (const definition of definitions) { + if (isRecord(definition) && typeof definition.name === 'string' && definition.name.trim()) { + collected.add(definition.name.trim()); + } + } + } + } + + return Array.from(collected); +} diff --git a/server/services/TaskSchedulerService/config.ts b/server/services/TaskSchedulerService/config.ts index 98b473c..d4cd71f 100644 --- a/server/services/TaskSchedulerService/config.ts +++ b/server/services/TaskSchedulerService/config.ts @@ -16,6 +16,14 @@ export const SLOT_HOLD_TIMEOUT_MS = parseInt( 10, ); +export const SCHEDULED_QUEUE_ITEM_TIMEOUT_MS = parseInt( + process.env.TASK_SCHEDULED_QUEUE_TIMEOUT_MS || String(Math.max( + QUEUE_ITEM_TIMEOUT_MS, + SLOT_HOLD_TIMEOUT_MS + 60 * 1000, + )), + 10, +); + export const SLOT_RECONCILE_INTERVAL_MS = parseInt( process.env.TASK_SLOT_RECONCILE_INTERVAL_MS || '5000', 10, diff --git a/server/services/TaskSchedulerService/directQueueController.ts b/server/services/TaskSchedulerService/directQueueController.ts new file mode 100644 index 0000000..dc1065c --- /dev/null +++ b/server/services/TaskSchedulerService/directQueueController.ts @@ -0,0 +1,263 @@ +import logger from '../../utils/logger'; +import { LOG_CONTEXTS, LOG_EVENTS } from '../../config/logging'; +import { + CONCURRENCY_LIMIT, + MAX_QUEUE_DEPTH, + QUEUE_ITEM_TIMEOUT_MS, + SLOT_HOLD_TIMEOUT_MS, +} from './config'; +import type { DirectQueueItem, RunningSlot } from './types'; + +interface DirectQueueControllerOptions { + runningSlots: Map; + directQueue: DirectQueueItem[]; + drainScheduledQueue: () => void; +} + +export class TaskSchedulerDirectQueueController { + private placeholderRunIdCounter = 0; + private readonly runningSlots: Map; + private readonly directQueue: DirectQueueItem[]; + private readonly drainScheduledQueue: () => void; + + constructor(options: DirectQueueControllerOptions) { + this.runningSlots = options.runningSlots; + this.directQueue = options.directQueue; + this.drainScheduledQueue = options.drainScheduledQueue; + } + + enqueueDirectJob(label: string, job: (placeholderRunId: number) => Promise): void { + if (this.runningSlots.size < CONCURRENCY_LIMIT) { + const placeholderRunId = this.nextPlaceholderRunId(); + const placeholderTimer = this.createPlaceholderTimer(placeholderRunId, label); + + this.runningSlots.set(placeholderRunId, { + taskId: 0, + runId: placeholderRunId, + startedAt: Date.now(), + timeoutTimer: placeholderTimer, + label: `placeholder:${label}`, + }); + + logger.debug(`[Direct] Slot pre-allocated immediately (placeholder=${placeholderRunId}) for ${label}`, { + event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, + slotsUsed: this.runningSlots.size, + limit: CONCURRENCY_LIMIT, + }, LOG_CONTEXTS.EXECUTION); + + setImmediate(() => job(placeholderRunId).catch(err => { + logger.errorLog(err, `[Direct] Immediate job failed for ${label}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }); + })); + return; + } + + if (this.directQueue.length >= MAX_QUEUE_DEPTH) { + logger.warn(`[Direct] Queue full, rejecting ${label}`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_FULL, + directQueueDepth: this.directQueue.length, + maxQueueDepth: MAX_QUEUE_DEPTH, + }, LOG_CONTEXTS.EXECUTION); + throw new Error(`并发执行队列已满(${MAX_QUEUE_DEPTH}),请稍后再试`); + } + + const item: DirectQueueItem = { + enqueuedAt: Date.now(), + label, + resolve: (placeholderRunId: number) => { + job(placeholderRunId).catch(err => { + logger.errorLog(err, `[Direct] Queued job failed for ${label}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }); + }); + }, + reject: (err: Error) => { + logger.warn(`[Direct] Queued job rejected for ${label}: ${err.message}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }, LOG_CONTEXTS.EXECUTION); + }, + }; + + item.timeoutTimer = setTimeout(() => { + const idx = this.directQueue.indexOf(item); + if (idx !== -1) { + this.directQueue.splice(idx, 1); + logger.warn(`[Direct] Queue item timed out for ${label}`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_TIMEOUT, + waitMs: Date.now() - item.enqueuedAt, + }, LOG_CONTEXTS.EXECUTION); + item.reject(new Error(`排队等待超时(${Math.round(QUEUE_ITEM_TIMEOUT_MS / 1000)}秒),任务已取消`)); + } + }, QUEUE_ITEM_TIMEOUT_MS); + if (item.timeoutTimer.unref) item.timeoutTimer.unref(); + + this.directQueue.push(item); + + logger.info(`[Direct] ${label} queued (concurrency limit ${CONCURRENCY_LIMIT} reached)`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUED, + label, + directQueuePosition: this.directQueue.length, + slotsUsed: this.runningSlots.size, + }, LOG_CONTEXTS.EXECUTION); + + setImmediate(() => this.drainDirectQueue()); + } + + async acquireDirectSlot(label: string): Promise { + if (this.runningSlots.size < CONCURRENCY_LIMIT) { + logger.debug(`[Direct] Slot acquired immediately for ${label}`, { + event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, + slotsUsed: this.runningSlots.size, + limit: CONCURRENCY_LIMIT, + }, LOG_CONTEXTS.EXECUTION); + return; + } + + if (this.directQueue.length >= MAX_QUEUE_DEPTH) { + logger.warn(`[Direct] Queue full, rejecting ${label}`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_FULL, + directQueueDepth: this.directQueue.length, + maxQueueDepth: MAX_QUEUE_DEPTH, + }, LOG_CONTEXTS.EXECUTION); + throw new Error(`并发执行队列已满(${MAX_QUEUE_DEPTH}),请稍后再试`); + } + + return new Promise((resolve, reject) => { + const item: DirectQueueItem = { + enqueuedAt: Date.now(), + label, + resolve: () => resolve(), + reject, + }; + + item.timeoutTimer = setTimeout(() => { + const idx = this.directQueue.indexOf(item); + if (idx !== -1) { + this.directQueue.splice(idx, 1); + logger.warn(`[Direct] Queue item timed out for ${label}`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_TIMEOUT, + waitMs: Date.now() - item.enqueuedAt, + }, LOG_CONTEXTS.EXECUTION); + reject(new Error(`Queue item timed out for ${label}`)); + } + }, QUEUE_ITEM_TIMEOUT_MS); + if (item.timeoutTimer.unref) item.timeoutTimer.unref(); + + this.directQueue.push(item); + + logger.info(`[Direct] ${label} queued (concurrency limit ${CONCURRENCY_LIMIT} reached)`, { + event: LOG_EVENTS.SCHEDULER_TASK_QUEUED, + label, + directQueuePosition: this.directQueue.length, + slotsUsed: this.runningSlots.size, + }, LOG_CONTEXTS.EXECUTION); + }); + } + + registerDirectSlot(runId: number, label: string, placeholderRunId?: number): void { + if (placeholderRunId !== undefined) { + const placeholder = this.runningSlots.get(placeholderRunId); + if (placeholder) { + clearTimeout(placeholder.timeoutTimer); + this.runningSlots.delete(placeholderRunId); + } + } + + const timeoutTimer = setTimeout(() => { + const slot = this.runningSlots.get(runId); + if (slot) { + this.runningSlots.delete(runId); + logger.warn(`[Direct] Slot for runId=${runId} (${label}) auto-released after timeout`, { + event: LOG_EVENTS.SCHEDULER_SLOT_TIMEOUT, + runId, + label, + heldMs: SLOT_HOLD_TIMEOUT_MS, + }, LOG_CONTEXTS.EXECUTION); + this.drainDirectQueue(); + this.drainScheduledQueue(); + } + }, SLOT_HOLD_TIMEOUT_MS); + + if (timeoutTimer.unref) timeoutTimer.unref(); + + this.runningSlots.set(runId, { + taskId: 0, + runId, + startedAt: Date.now(), + timeoutTimer, + label, + }); + + logger.info(`[Direct] Slot registered for runId=${runId} (${label})`, { + event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, + runId, + label, + slotsUsed: this.runningSlots.size, + slotsLimit: CONCURRENCY_LIMIT, + }, LOG_CONTEXTS.EXECUTION); + } + + drainDirectQueue(): void { + while (this.directQueue.length > 0 && this.runningSlots.size < CONCURRENCY_LIMIT) { + const next = this.directQueue.shift()!; + if (next.timeoutTimer) clearTimeout(next.timeoutTimer); + + const placeholderRunId = this.nextPlaceholderRunId(); + const placeholderTimer = this.createPlaceholderTimer(placeholderRunId, next.label); + + this.runningSlots.set(placeholderRunId, { + taskId: 0, + runId: placeholderRunId, + startedAt: Date.now(), + timeoutTimer: placeholderTimer, + label: `placeholder:${next.label}`, + }); + + logger.debug(`[Direct] Draining queue: slot pre-allocated (placeholder=${placeholderRunId}) for ${next.label} (waited ${Date.now() - next.enqueuedAt}ms)`, { + event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, + label: next.label, + placeholderRunId, + directQueueDepth: this.directQueue.length, + slotsUsed: this.runningSlots.size, + }, LOG_CONTEXTS.EXECUTION); + + next.resolve(placeholderRunId); + } + } + + releaseSlotByRunId(runId: number, source: 'callback' | 'db_reconcile' | 'db_missing' = 'callback'): void { + const slot = this.runningSlots.get(runId); + if (!slot || runId < 0) return; + + clearTimeout(slot.timeoutTimer); + this.runningSlots.delete(runId); + + logger.info(`[P1] Slot released for runId=${runId} (taskId=${slot.taskId}) via ${source}`, { + event: LOG_EVENTS.SCHEDULER_SLOT_RELEASED, + runId, + taskId: slot.taskId, + source, + heldMs: Date.now() - slot.startedAt, + }, LOG_CONTEXTS.EXECUTION); + + this.drainDirectQueue(); + this.drainScheduledQueue(); + } + + private nextPlaceholderRunId(): number { + this.placeholderRunIdCounter -= 1; + return this.placeholderRunIdCounter; + } + + private createPlaceholderTimer(placeholderRunId: number, label: string): NodeJS.Timeout { + const placeholderTimer = setTimeout(() => { + if (this.runningSlots.has(placeholderRunId)) { + this.runningSlots.delete(placeholderRunId); + logger.warn(`[Direct] Placeholder slot ${placeholderRunId} for ${label} expired, releasing`, { + event: LOG_EVENTS.SCHEDULER_SLOT_TIMEOUT, + label, + }, LOG_CONTEXTS.EXECUTION); + this.drainDirectQueue(); + this.drainScheduledQueue(); + } + }, 30_000); + + if (placeholderTimer.unref) placeholderTimer.unref(); + return placeholderTimer; + } +} diff --git a/server/services/TaskSchedulerService/queuePolicy.ts b/server/services/TaskSchedulerService/queuePolicy.ts new file mode 100644 index 0000000..8b399a1 --- /dev/null +++ b/server/services/TaskSchedulerService/queuePolicy.ts @@ -0,0 +1,45 @@ +import { + QUEUE_ITEM_TIMEOUT_MS, + SCHEDULED_QUEUE_ITEM_TIMEOUT_MS, +} from './config'; +import type { QueueItem } from './types'; + +type TriggerReason = QueueItem['triggerReason']; +type QueueIdentity = Pick; + +function scheduledWindowMs(value?: Date): number | null { + if (!value) return null; + const time = value.getTime(); + return Number.isNaN(time) ? null : time; +} + +export function getQueueItemTimeoutMs(triggerReason: TriggerReason): number { + return triggerReason === 'scheduled' + ? SCHEDULED_QUEUE_ITEM_TIMEOUT_MS + : QUEUE_ITEM_TIMEOUT_MS; +} + +export function isDuplicateQueuedDispatch( + existing: QueueIdentity, + candidate: QueueIdentity, +): boolean { + if (existing.taskId !== candidate.taskId) { + return false; + } + + if (candidate.triggerReason === 'scheduled') { + if (existing.triggerReason !== 'scheduled') { + return false; + } + + const existingWindowMs = scheduledWindowMs(existing.scheduledFor); + const candidateWindowMs = scheduledWindowMs(candidate.scheduledFor); + if (existingWindowMs === null || candidateWindowMs === null) { + return true; + } + + return existingWindowMs === candidateWindowMs; + } + + return true; +} diff --git a/server/services/TaskSchedulerService/registry.ts b/server/services/TaskSchedulerService/registry.ts index e0372cb..095bf32 100644 --- a/server/services/TaskSchedulerService/registry.ts +++ b/server/services/TaskSchedulerService/registry.ts @@ -21,7 +21,12 @@ import type { ScheduledTask } from './types'; interface TaskSchedulerRegistryHelperDeps { taskCache: Map; timers: Map; - dispatchTask: (taskId: number, triggerReason?: 'scheduled' | 'manual' | 'retry', operatorId?: number) => Promise; + dispatchTask: ( + taskId: number, + triggerReason?: 'scheduled' | 'manual' | 'retry', + operatorId?: number, + scheduledFor?: Date, + ) => Promise; recordAuditLog: (taskId: number, action: string, operatorId: number | null, metadata: Record) => Promise; unregisterTask: (taskId: number) => void; } @@ -96,7 +101,7 @@ export class TaskSchedulerRegistryHelper { prevShouldRun: prevShouldRun.toISOString(), lastRunAt: task.lastRunAt?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); - this.deps.dispatchTask(task.id, 'scheduled').catch((err) => { + this.deps.dispatchTask(task.id, 'scheduled', undefined, prevShouldRun).catch((err) => { logger.errorLog(err, `Compensation dispatch failed for task ${task.id}`, { event: LOG_EVENTS.SCHEDULER_TASK_COMPENSATION_FAILED }); }); }); @@ -159,7 +164,7 @@ export class TaskSchedulerRegistryHelper { dueAt: next.toISOString(), }, LOG_CONTEXTS.EXECUTION); - await this.deps.dispatchTask(task.id, 'scheduled').catch((err) => { + await this.deps.dispatchTask(task.id, 'scheduled', undefined, next).catch((err) => { logger.errorLog(err, `Scheduled dispatch failed for task ${task.id}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED }); }); diff --git a/server/services/TaskSchedulerService/scheduledWindowDedupe.ts b/server/services/TaskSchedulerService/scheduledWindowDedupe.ts new file mode 100644 index 0000000..ad63847 --- /dev/null +++ b/server/services/TaskSchedulerService/scheduledWindowDedupe.ts @@ -0,0 +1,157 @@ +import { queryOne } from '../../config/database'; +import logger from '../../utils/logger'; +import { LOG_CONTEXTS, LOG_EVENTS } from '../../config/logging'; +import { MAX_MISSED_WINDOW_MS } from './config'; +import { getPrevCronTime } from './cron'; + +export interface ScheduledWindowDedupeInput { + taskId: number; + cronExpression: string; + scheduledFor?: Date; + recentScheduledWindowsByTaskId: Map>; +} + +export interface ScheduledWindowDedupeResult { + duplicated: boolean; + windowStart: Date | null; + reason?: string; +} + +function rememberScheduledWindow( + recentScheduledWindowsByTaskId: Map>, + taskId: number, + windowStartMs: number +): void { + let windows = recentScheduledWindowsByTaskId.get(taskId); + if (!windows) { + windows = new Set(); + recentScheduledWindowsByTaskId.set(taskId, windows); + } + + const cutoffMs = Date.now() - MAX_MISSED_WINDOW_MS; + for (const rememberedWindow of windows) { + if (rememberedWindow < cutoffMs) { + windows.delete(rememberedWindow); + } + } + + windows.add(windowStartMs); +} + +function hasRememberedScheduledWindow( + recentScheduledWindowsByTaskId: Map>, + taskId: number, + windowStartMs: number +): boolean { + const windows = recentScheduledWindowsByTaskId.get(taskId); + return windows?.has(windowStartMs) ?? false; +} + +export async function isDuplicateScheduledWindow({ + taskId, + cronExpression, + scheduledFor, + recentScheduledWindowsByTaskId, +}: ScheduledWindowDedupeInput): Promise { + const now = new Date(); + const windowStart = scheduledFor && !Number.isNaN(scheduledFor.getTime()) + ? scheduledFor + : getPrevCronTime( + cronExpression, + new Date(now.getTime() + 1000), + MAX_MISSED_WINDOW_MS, + ); + + if (!windowStart) { + logger.debug(`Unable to compute previous cron window for task ${taskId}`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + cronExpression, + scheduledFor: scheduledFor?.toISOString() ?? null, + }, LOG_CONTEXTS.EXECUTION); + return { duplicated: false, windowStart: null, reason: 'window_unavailable' }; + } + + const windowStartMs = windowStart.getTime(); + const windowStartIso = windowStart.toISOString(); + + if (hasRememberedScheduledWindow(recentScheduledWindowsByTaskId, taskId, windowStartMs)) { + logger.info(`Task ${taskId} duplicate detected by memory guard`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + windowStart: windowStartIso, + reason: 'memory_guard', + }, LOG_CONTEXTS.EXECUTION); + return { duplicated: true, windowStart, reason: 'memory_guard' }; + } + + rememberScheduledWindow(recentScheduledWindowsByTaskId, taskId, windowStartMs); + + let auditMatch: { id: number } | null = null; + try { + auditMatch = await queryOne<{ id: number }>( + `SELECT id + FROM Auto_TaskAuditLogs + WHERE task_id = ? + AND action = 'triggered' + AND JSON_VALID(metadata) + AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.scheduledFor')) = ? + ORDER BY created_at DESC + LIMIT 1`, + [taskId, windowStartIso], + ); + } catch (err) { + logger.warn(`Task ${taskId} scheduled window audit guard failed, continuing with memory guard`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + windowStart: windowStartIso, + error: err instanceof Error ? err.message : String(err), + }, LOG_CONTEXTS.EXECUTION); + } + + if (auditMatch) { + logger.info(`Task ${taskId} duplicate detected by scheduled window audit guard`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + windowStart: windowStartIso, + reason: 'audit_window_guard', + }, LOG_CONTEXTS.EXECUTION); + return { duplicated: true, windowStart, reason: 'audit_window_guard' }; + } + + if (!scheduledFor) { + const lastExecution = await queryOne<{ id: number; created_at: string | null }>( + `SELECT id, created_at + FROM Auto_TestCaseTaskExecutions + WHERE task_id = ? + ORDER BY created_at DESC + LIMIT 1`, + [taskId], + ); + + if (lastExecution?.created_at) { + const lastCreatedAt = new Date(lastExecution.created_at).getTime(); + const toleranceMs = parseInt(process.env.SCHEDULER_DEDUPE_TOLERANCE_MS || '5000', 10); + if (!Number.isNaN(lastCreatedAt) && lastCreatedAt >= (windowStartMs - toleranceMs)) { + logger.info(`Task ${taskId} duplicate detected by DB guard`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + windowStart: windowStartIso, + lastExecutionCreatedAt: new Date(lastCreatedAt).toISOString(), + toleranceMs, + reason: 'db_guard', + }, LOG_CONTEXTS.EXECUTION); + return { duplicated: true, windowStart, reason: 'db_guard' }; + } + } + } else { + logger.debug(`Task ${taskId} skipped legacy DB duplicate guard for explicit scheduled window`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + windowStart: windowStartIso, + reason: 'explicit_scheduled_window', + }, LOG_CONTEXTS.EXECUTION); + } + + return { duplicated: false, windowStart }; +} diff --git a/server/services/TaskSchedulerService/service.ts b/server/services/TaskSchedulerService/service.ts index fe34367..de26f18 100644 --- a/server/services/TaskSchedulerService/service.ts +++ b/server/services/TaskSchedulerService/service.ts @@ -1,7 +1,6 @@ /** * TaskSchedulerService - 任务定时调度引擎 * - * 功能: * 1. 启动时加载所有 scheduled + active 任务,注册 Cron 定时器 * 2. 支持动态添加/移除/更新任务调度 * 3. 服务启动恢复:检测漏触发(上次执行时间 + cron 下次执行时间 < now),进行补偿执行 @@ -9,10 +8,8 @@ * 5. 失败重试:支持 maxRetries / retryDelayMs 配置 * 6. [P1] 并发槽位以 runId 为维度,槽位在 Jenkins 回调或执行超时后释放 * 7. [P1] 等待队列支持优先级、入队时间、最大深度和队列超时机制 - * * 依赖:croner(零依赖、原生 TS 支持的 cron 解析库) */ - import { query, queryOne, getPool } from '../../config/database'; import logger from '../../utils/logger'; import { LOG_CONTEXTS, LOG_EVENTS } from '../../config/logging'; @@ -32,11 +29,12 @@ import { PRIORITY_RETRY, PRIORITY_SCHEDULED, QUEUE_ITEM_TIMEOUT_MS, + SCHEDULED_QUEUE_ITEM_TIMEOUT_MS, SCHEDULER_USER_ID, SLOT_HOLD_TIMEOUT_MS, SLOT_RECONCILE_INTERVAL_MS, } from './config'; -import { getNextCronTime, getPrevCronTime } from './cron'; +import { getNextCronTime } from './cron'; import { loadAllScheduledTasks, loadLastRunAt, @@ -45,6 +43,13 @@ import { mapPollRowToScheduledTask, } from './taskQueries'; import { TaskSchedulerRegistryHelper } from './registry'; +import { + getQueueItemTimeoutMs, + isDuplicateQueuedDispatch, +} from './queuePolicy'; +import { TaskSchedulerDirectQueueController } from './directQueueController'; +import { isDuplicateScheduledWindow } from './scheduledWindowDedupe'; +import { buildSchedulerStatus, type TaskSchedulerStatusSnapshot } from './status'; import type { DirectQueueItem, QueueItem, @@ -52,13 +57,10 @@ import type { RunningSlot, ScheduledTask, } from './types'; -// ────────────────────────────────────────────────────────── -// 类型定义 -// ────────────────────────────────────────────────────────── - export class TaskSchedulerService { private readonly registryHelper: TaskSchedulerRegistryHelper; + private readonly directQueueController: TaskSchedulerDirectQueueController; /** taskId → setInterval / setTimeout handle */ private timers: Map = new Map(); @@ -88,16 +90,14 @@ export class TaskSchedulerService { private taskCache: Map = new Map(); /** - * 最近一次已触发的 cron 窗口(taskId -> windowStartMs) + * 最近已触发的 cron 窗口(taskId -> windowStartMs set) * 用于防止“补偿触发 + 正常定时触发”在同一窗口内重复创建运行记录。 */ - private recentScheduledWindowByTaskId: Map = new Map(); + private recentScheduledWindowsByTaskId: Map> = new Map(); /** 是否已启动 */ private started = false; - /** 占位 runId 计数器(负数,递减,保证不与真实 runId 冲突) */ - private _placeholderRunIdCounter = 0; /** 周期轮询 DB 同步任务变更的定时器 */ private taskPollTimer?: NodeJS.Timeout; @@ -109,10 +109,16 @@ export class TaskSchedulerService { private slotReconcileInFlight = false; constructor() { + this.directQueueController = new TaskSchedulerDirectQueueController({ + runningSlots: this.runningSlots, + directQueue: this.directQueue, + drainScheduledQueue: () => this.drainQueue(), + }); + this.registryHelper = new TaskSchedulerRegistryHelper({ taskCache: this.taskCache, timers: this.timers, - dispatchTask: (taskId, triggerReason, operatorId) => this.dispatchTask(taskId, triggerReason, operatorId), + dispatchTask: (taskId, triggerReason, operatorId, scheduledFor) => this.dispatchTask(taskId, triggerReason, operatorId, scheduledFor), recordAuditLog: (taskId, action, operatorId, metadata) => this.recordAuditLog(taskId, action, operatorId, metadata), unregisterTask: (taskId) => this.unregisterTask(taskId), }); @@ -181,6 +187,8 @@ export class TaskSchedulerService { logger.info('TaskSchedulerService started', { event: LOG_EVENTS.SCHEDULER_STARTED, concurrencyLimit: CONCURRENCY_LIMIT, + queueItemTimeoutMs: QUEUE_ITEM_TIMEOUT_MS, + scheduledQueueItemTimeoutMs: SCHEDULED_QUEUE_ITEM_TIMEOUT_MS, slotReconcileIntervalMs: SLOT_RECONCILE_INTERVAL_MS, }, LOG_CONTEXTS.EXECUTION); } @@ -356,6 +364,7 @@ export class TaskSchedulerService { if (!task) return; this.unregisterTask(taskId); if (task.status === 'active' && task.cronExpression) { + this.taskCache.set(task.id, task); this.registryHelper.scheduleTask(task); } } @@ -369,7 +378,7 @@ export class TaskSchedulerService { this.timers.delete(taskId); } this.taskCache.delete(taskId); - this.recentScheduledWindowByTaskId.delete(taskId); + this.recentScheduledWindowsByTaskId.delete(taskId); // [P1] 从等待队列中也清除,并取消队列超时 timer const removedItems = this.waitQueue.filter(item => item.taskId === taskId); for (const item of removedItems) { @@ -379,356 +388,38 @@ export class TaskSchedulerService { logger.debug(`Task ${taskId} unregistered from scheduler`, { event: LOG_EVENTS.SCHEDULER_TASK_UNREGISTERED, taskId }, LOG_CONTEXTS.EXECUTION); } - /** - * [P1] 获取调度器当前状态(扩展版,供监控接口使用) - * 新增:队列深度、每个队列项的排队时长、每个运行槽位已运行时长 - */ - getStatus(): { - running: Array<{ taskId: number; runId: number; elapsedMs: number; label?: string }>; - queued: Array<{ taskId: number; triggerReason: string; waitMs: number; priority: number; queuePosition: number }>; - directQueued: Array<{ label: string; waitMs: number; queuePosition: number }>; - scheduled: number[]; - concurrencyLimit: number; - queueDepth: number; - directQueueDepth: number; - maxQueueDepth: number; - } { - const now = Date.now(); - return { - running: Array.from(this.runningSlots.values()).map(slot => ({ - taskId: slot.taskId, - runId: slot.runId, - elapsedMs: now - slot.startedAt, - label: slot.label, - })), - queued: this.waitQueue.map((item, idx) => ({ - taskId: item.taskId, - triggerReason: item.triggerReason, - waitMs: now - item.enqueuedAt, - priority: item.priority, - queuePosition: idx + 1, - })), - directQueued: this.directQueue.map((item, idx) => ({ - label: item.label, - waitMs: now - item.enqueuedAt, - queuePosition: idx + 1, - })), - scheduled: Array.from(this.taskCache.keys()), - concurrencyLimit: CONCURRENCY_LIMIT, - queueDepth: this.waitQueue.length, - directQueueDepth: this.directQueue.length, - maxQueueDepth: MAX_QUEUE_DEPTH, - }; + getStatus(): TaskSchedulerStatusSnapshot { + return buildSchedulerStatus({ + runningSlots: this.runningSlots, + waitQueue: this.waitQueue, + directQueue: this.directQueue, + taskCache: this.taskCache, + }); } - - /** - * [P1] 查询某个任务在队列中的位置(供前端显示"排队中"状态) - * @returns 1-based position,0 表示不在队列中 - */ getQueuePosition(taskId: number): number { const idx = this.waitQueue.findIndex(item => item.taskId === taskId); return idx === -1 ? 0 : idx + 1; } - /** - * 异步直连入队(run-case / run-batch 推荐使用) - * - * 行为: - * - 立即返回(非阻塞),不等待槽位 - * - 若当前槽位有空余,则 setImmediate 后异步调用 job() - * - 若槽位满,则将 job 放入 directQueue,待槽位释放后自动触发 - * - 队列已满(MAX_QUEUE_DEPTH)→ 抛出错误(调用方返回 503) - * - * @param label 来源标识(如 "case:123"),用于日志和监控展示 - * @param job 异步任务回调,槽位可用时执行;接收占位 runId(负数), - * job 内部应调用 registerDirectSlot(realRunId, label, placeholderRunId) 替换占位 - */ enqueueDirectJob(label: string, job: (placeholderRunId: number) => Promise): void { - if (this.runningSlots.size < CONCURRENCY_LIMIT) { - // 有空余槽位,立即预占一个槽位,再异步执行 job - const placeholderRunId = --this._placeholderRunIdCounter; - const placeholderTimer = setTimeout(() => { - if (this.runningSlots.has(placeholderRunId)) { - this.runningSlots.delete(placeholderRunId); - logger.warn(`[Direct] Placeholder slot ${placeholderRunId} for ${label} expired (immediate path)`, { - event: LOG_EVENTS.SCHEDULER_SLOT_TIMEOUT, - label, - }, LOG_CONTEXTS.EXECUTION); - this.drainDirectQueue(); - this.drainQueue(); - } - }, 30_000); - if (placeholderTimer.unref) placeholderTimer.unref(); - - this.runningSlots.set(placeholderRunId, { - taskId: 0, - runId: placeholderRunId, - startedAt: Date.now(), - timeoutTimer: placeholderTimer, - label: `placeholder:${label}`, - }); - - logger.debug(`[Direct] Slot pre-allocated immediately (placeholder=${placeholderRunId}) for ${label}`, { - event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, - slotsUsed: this.runningSlots.size, - limit: CONCURRENCY_LIMIT, - }, LOG_CONTEXTS.EXECUTION); - - setImmediate(() => job(placeholderRunId).catch(err => { - logger.errorLog(err, `[Direct] Immediate job failed for ${label}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }); - })); - return; - } - - // 队列深度保护 - if (this.directQueue.length >= MAX_QUEUE_DEPTH) { - logger.warn(`[Direct] Queue full, rejecting ${label}`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_FULL, - directQueueDepth: this.directQueue.length, - maxQueueDepth: MAX_QUEUE_DEPTH, - }, LOG_CONTEXTS.EXECUTION); - throw new Error(`并发执行队列已满(${MAX_QUEUE_DEPTH}),请稍后再试`); - } - - // 包装成 DirectQueueItem,resolve(placeholderRunId) 时执行 job - const item: DirectQueueItem = { - enqueuedAt: Date.now(), - label, - resolve: (placeholderRunId: number) => { - job(placeholderRunId).catch(err => { - logger.errorLog(err, `[Direct] Queued job failed for ${label}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }); - }); - }, - reject: (err: Error) => { - logger.warn(`[Direct] Queued job rejected for ${label}: ${err.message}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED, label }, LOG_CONTEXTS.EXECUTION); - }, - }; - - // 队列超时自动移除(超时后不再执行,但 runId 已创建,状态会因无回调而由 slot 超时处理) - item.timeoutTimer = setTimeout(() => { - const idx = this.directQueue.indexOf(item); - if (idx !== -1) { - this.directQueue.splice(idx, 1); - logger.warn(`[Direct] Queue item timed out for ${label}`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_TIMEOUT, - waitMs: Date.now() - item.enqueuedAt, - }, LOG_CONTEXTS.EXECUTION); - item.reject(new Error(`排队等待超时(${Math.round(QUEUE_ITEM_TIMEOUT_MS / 1000)}秒),任务已取消`)); - } - }, QUEUE_ITEM_TIMEOUT_MS); - if (item.timeoutTimer.unref) item.timeoutTimer.unref(); - - this.directQueue.push(item); - - logger.info(`[Direct] ${label} queued (concurrency limit ${CONCURRENCY_LIMIT} reached)`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUED, - label, - directQueuePosition: this.directQueue.length, - slotsUsed: this.runningSlots.size, - }, LOG_CONTEXTS.EXECUTION); - - // 入队后立即尝试一次 drain,防止入队时恰好有槽位释放的竞态 - setImmediate(() => this.drainDirectQueue()); + this.directQueueController.enqueueDirectJob(label, job); } - /** - * 直连槽位申请(run-case / run-batch 专用) - * - * 行为: - * - 当前 runningSlots 未满 → 立即返回(可立即执行) - * - 已满但队列未满 → 排队等待,直到有槽位释放(Promise 在槽位可用时 resolve) - * - 队列已满 → 立即 reject(返回 503) - * - 等待超时(QUEUE_ITEM_TIMEOUT_MS)→ 自动 reject - * - * 注意:调用方必须在执行完成(或失败)后调用 releaseDirectSlot(runId) 释放槽位。 - * - * @param label 来源标识(如 "case:123"),用于日志和监控展示 - * @returns Promise,resolve 时可以开始执行 - */ async acquireDirectSlot(label: string): Promise { - if (this.runningSlots.size < CONCURRENCY_LIMIT) { - logger.debug(`[Direct] Slot acquired immediately for ${label}`, { - event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, - slotsUsed: this.runningSlots.size, - limit: CONCURRENCY_LIMIT, - }, LOG_CONTEXTS.EXECUTION); - return; // 直接通过 - } - - // 队列深度保护 - if (this.directQueue.length >= MAX_QUEUE_DEPTH) { - logger.warn(`[Direct] Queue full, rejecting ${label}`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_FULL, - directQueueDepth: this.directQueue.length, - maxQueueDepth: MAX_QUEUE_DEPTH, - }, LOG_CONTEXTS.EXECUTION); - throw new Error(`并发执行队列已满(${MAX_QUEUE_DEPTH}),请稍后再试`); - } - - // 排队等待 - return new Promise((resolve, reject) => { - const item: DirectQueueItem = { - enqueuedAt: Date.now(), - label, - resolve: (_placeholderRunId: number) => resolve(), - reject, - }; - - // 队列超时自动移除 - item.timeoutTimer = setTimeout(() => { - const idx = this.directQueue.indexOf(item); - if (idx !== -1) { - this.directQueue.splice(idx, 1); - logger.warn(`[Direct] Queue item timed out for ${label}`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_TIMEOUT, - waitMs: Date.now() - item.enqueuedAt, - }, LOG_CONTEXTS.EXECUTION); - reject(new Error(`Queue item timed out for ${label}`)); - } - }, QUEUE_ITEM_TIMEOUT_MS); - if (item.timeoutTimer.unref) item.timeoutTimer.unref(); - - this.directQueue.push(item); - - logger.info(`[Direct] ${label} queued (concurrency limit ${CONCURRENCY_LIMIT} reached)`, { - event: LOG_EVENTS.SCHEDULER_TASK_QUEUED, - label, - directQueuePosition: this.directQueue.length, - slotsUsed: this.runningSlots.size, - }, LOG_CONTEXTS.EXECUTION); - }); + return this.directQueueController.acquireDirectSlot(label); } - /** - * 直连槽位注册(enqueueDirectJob job 内部调用,将真实 runId 替换占位槽位) - * 槽位持有至 Jenkins 回调(releaseSlotByRunId)或超时自动释放 - * - * @param runId 真实的执行 runId(正数) - * @param label 来源标识 - * @param placeholderRunId 占位 runId(负数),注册前先删除占位槽位;不传则不删 - */ registerDirectSlot(runId: number, label: string, placeholderRunId?: number): void { - // 移除占位槽位(释放占位的超时 timer) - if (placeholderRunId !== undefined) { - const placeholder = this.runningSlots.get(placeholderRunId); - if (placeholder) { - clearTimeout(placeholder.timeoutTimer); - this.runningSlots.delete(placeholderRunId); - } - } - const timeoutTimer = setTimeout(() => { - const slot = this.runningSlots.get(runId); - if (slot) { - this.runningSlots.delete(runId); - logger.warn(`[Direct] Slot for runId=${runId} (${label}) auto-released after timeout`, { - event: LOG_EVENTS.SCHEDULER_SLOT_TIMEOUT, - runId, - label, - heldMs: SLOT_HOLD_TIMEOUT_MS, - }, LOG_CONTEXTS.EXECUTION); - this.drainDirectQueue(); - this.drainQueue(); - } - }, SLOT_HOLD_TIMEOUT_MS); - - if (timeoutTimer.unref) timeoutTimer.unref(); - - this.runningSlots.set(runId, { - taskId: 0, // 直连执行无 taskId - runId, - startedAt: Date.now(), - timeoutTimer, - label, - }); - - logger.info(`[Direct] Slot registered for runId=${runId} (${label})`, { - event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, - runId, - label, - slotsUsed: this.runningSlots.size, - slotsLimit: CONCURRENCY_LIMIT, - }, LOG_CONTEXTS.EXECUTION); + this.directQueueController.registerDirectSlot(runId, label, placeholderRunId); } - /** - * 从直连等待队列中取出下一个等待方并通知(FIFO) - * 在槽位释放后(releaseSlotByRunId / drainQueue)调用 - * - * 关键设计:drain 出一个 job 后,立即用占位 runId(负数)预占一个槽位, - * 防止 while 循环在 job 异步执行前重复 drain(竞态条件)。 - * job 内部调用 registerDirectSlot(realRunId) 时会覆盖占位槽位。 - */ private drainDirectQueue(): void { - while (this.directQueue.length > 0 && this.runningSlots.size < CONCURRENCY_LIMIT) { - const next = this.directQueue.shift()!; - if (next.timeoutTimer) clearTimeout(next.timeoutTimer); - - // 用占位 runId(负数,递减保证唯一)立即预占槽位,防止并发多取 - const placeholderRunId = --this._placeholderRunIdCounter; - const placeholderTimer = setTimeout(() => { - // 占位超时(正常情况下 registerDirectSlot 会在几毫秒内替换掉占位) - if (this.runningSlots.has(placeholderRunId)) { - this.runningSlots.delete(placeholderRunId); - logger.warn(`[Direct] Placeholder slot ${placeholderRunId} for ${next.label} expired, releasing`, { - event: LOG_EVENTS.SCHEDULER_SLOT_TIMEOUT, - label: next.label, - }, LOG_CONTEXTS.EXECUTION); - this.drainDirectQueue(); - this.drainQueue(); - } - }, 30_000); // 30秒兜底 - if (placeholderTimer.unref) placeholderTimer.unref(); - - this.runningSlots.set(placeholderRunId, { - taskId: 0, - runId: placeholderRunId, - startedAt: Date.now(), - timeoutTimer: placeholderTimer, - label: `placeholder:${next.label}`, - }); - - logger.debug(`[Direct] Draining queue: slot pre-allocated (placeholder=${placeholderRunId}) for ${next.label} (waited ${Date.now() - next.enqueuedAt}ms)`, { - event: LOG_EVENTS.SCHEDULER_SLOT_REGISTERED, - label: next.label, - placeholderRunId, - directQueueDepth: this.directQueue.length, - slotsUsed: this.runningSlots.size, - }, LOG_CONTEXTS.EXECUTION); - - // 通知 job 执行;job 内部应调用 registerDirectSlot(realRunId) 替换占位槽位 - next.resolve(placeholderRunId); - } + this.directQueueController.drainDirectQueue(); } - /** - * [P1] 释放运行槽位 - * 默认在 Jenkins 回调成功后调用;也可由对账任务兜底释放 - */ releaseSlotByRunId(runId: number, source: 'callback' | 'db_reconcile' | 'db_missing' = 'callback'): void { - const slot = this.runningSlots.get(runId); - // 不允许通过此方法释放占位槽位(负数 runId),占位由 registerDirectSlot 内部管理 - if (!slot || runId < 0) return; - - clearTimeout(slot.timeoutTimer); - this.runningSlots.delete(runId); - - logger.info(`[P1] Slot released for runId=${runId} (taskId=${slot.taskId}) via ${source}`, { - event: LOG_EVENTS.SCHEDULER_SLOT_RELEASED, - runId, - taskId: slot.taskId, - source, - heldMs: Date.now() - slot.startedAt, - }, LOG_CONTEXTS.EXECUTION); - - // 槽位释放后,尝试从队列中取出下一个任务(任务队列和直连队列都需要 drain) - this.drainDirectQueue(); - this.drainQueue(); + this.directQueueController.releaseSlotByRunId(runId, source); } - - // ───────────────────────────────────────────── - // 内部:加载与注册 - // ───────────────────────────────────────────── - private async loadAndRegisterAllTasks(): Promise { await this.registryHelper.loadAndRegisterAllTasks(); } @@ -770,7 +461,12 @@ export class TaskSchedulerService { * 3. 槽位在 executeTask 返回后**不立即释放**,而是在 Jenkins 回调/超时后释放 * 4. 队列最大深度保护(MAX_QUEUE_DEPTH)+ 队列项超时(QUEUE_ITEM_TIMEOUT_MS) */ - async dispatchTask(taskId: number, triggerReason: 'scheduled' | 'manual' | 'retry' = 'scheduled', operatorId?: number): Promise { + async dispatchTask( + taskId: number, + triggerReason: 'scheduled' | 'manual' | 'retry' = 'scheduled', + operatorId?: number, + scheduledFor?: Date, + ): Promise { if (this.runningSlots.size >= CONCURRENCY_LIMIT) { // [P1] 队列深度保护 if (this.waitQueue.length >= MAX_QUEUE_DEPTH) { @@ -783,9 +479,20 @@ export class TaskSchedulerService { return; } - // [P1] 同一任务已在队列中则跳过(防重复入队) - if (this.waitQueue.some(item => item.taskId === taskId)) { - logger.debug(`Task ${taskId} already in queue, skipping duplicate enqueue`, { event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, taskId }, LOG_CONTEXTS.EXECUTION); + const candidateQueueItem = { + taskId, + triggerReason, + scheduledFor, + }; + + // [P1] 同一调度窗口已在队列中则跳过;不同 cron 窗口需要保留,避免周期触发被吞掉。 + if (this.waitQueue.some(item => isDuplicateQueuedDispatch(item, candidateQueueItem))) { + logger.debug(`Task ${taskId} already in queue, skipping duplicate enqueue`, { + event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, + taskId, + triggerReason, + scheduledFor: scheduledFor?.toISOString() ?? null, + }, LOG_CONTEXTS.EXECUTION); return; } @@ -798,22 +505,27 @@ export class TaskSchedulerService { taskId, triggerReason, operatorId, + scheduledFor, enqueuedAt: Date.now(), priority, }; - // [P1] 队列项超时:超过 QUEUE_ITEM_TIMEOUT_MS 自动移除 + const queueTimeoutMs = getQueueItemTimeoutMs(triggerReason); + + // [P1] 队列项超时:scheduled 使用更长等待,避免周期触发在槽位释放前被丢弃 queueItem.timeoutTimer = setTimeout(() => { const idx = this.waitQueue.findIndex(it => it === queueItem); if (idx !== -1) { this.waitQueue.splice(idx, 1); - logger.warn(`Task ${taskId} queue item timed out and removed (waited ${QUEUE_ITEM_TIMEOUT_MS}ms)`, { + logger.warn(`Task ${taskId} queue item timed out and removed (waited ${queueTimeoutMs}ms)`, { event: LOG_EVENTS.SCHEDULER_TASK_QUEUE_TIMEOUT, taskId, waitMs: Date.now() - queueItem.enqueuedAt, + triggerReason, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); } - }, QUEUE_ITEM_TIMEOUT_MS); + }, queueTimeoutMs); if (queueItem.timeoutTimer.unref) queueItem.timeoutTimer.unref(); // [P1] 按优先级插入(稳定排序:priority ASC,enqueuedAt ASC) @@ -833,6 +545,8 @@ export class TaskSchedulerService { priority, queuePosition: this.waitQueue.findIndex(it => it === queueItem) + 1, queueDepth: this.waitQueue.length, + queueTimeoutMs, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); return; } @@ -843,11 +557,12 @@ export class TaskSchedulerService { triggerReason, runningSlots: this.runningSlots.size, limit: CONCURRENCY_LIMIT, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); try { // [P1] executeTask 内部会在 Jenkins 触发成功后注册 slot,失败时不注册(不占用槽位) - await this.executeTask(taskId, triggerReason, operatorId); + await this.executeTask(taskId, triggerReason, operatorId, scheduledFor); // 触发成功后清理历史重试状态,避免后续误判 this.clearRetryState(taskId); } catch (err) { @@ -860,72 +575,10 @@ export class TaskSchedulerService { // 注意:不再在 finally 中释放槽位!槽位由 releaseSlotByRunId 或超时 timer 负责释放 } - /** - * 判断当前 scheduled 触发是否与最近窗口重复(补偿 + 定时并发场景防重) - */ - private async isDuplicateScheduledWindow(taskId: number, cronExpression: string): Promise<{ duplicated: boolean; windowStart: Date | null; reason?: string }> { - const now = new Date(); - const windowStart = getPrevCronTime( - cronExpression, - new Date(now.getTime() + 1000), - MAX_MISSED_WINDOW_MS, - ); - if (!windowStart) { - logger.debug(`Unable to compute previous cron window for task ${taskId}`, { - event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, - taskId, - cronExpression, - }, LOG_CONTEXTS.EXECUTION); - return { duplicated: false, windowStart: null, reason: 'window_unavailable' }; - } - - const windowStartMs = windowStart.getTime(); - const inMemoryWindow = this.recentScheduledWindowByTaskId.get(taskId); - if (inMemoryWindow === windowStartMs) { - logger.info(`Task ${taskId} duplicate detected by memory guard`, { - event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, - taskId, - windowStart: new Date(windowStartMs).toISOString(), - reason: 'memory_guard', - }, LOG_CONTEXTS.EXECUTION); - return { duplicated: true, windowStart, reason: 'memory_guard' }; - } - - const lastExecution = await queryOne<{ id: number; created_at: string | null }>( - `SELECT id, created_at - FROM Auto_TestCaseTaskExecutions - WHERE task_id = ? - ORDER BY created_at DESC - LIMIT 1`, - [taskId] - ); - - if (lastExecution?.created_at) { - const lastCreatedAt = new Date(lastExecution.created_at).getTime(); - // 容错:允许少量时钟偏移(默认 5s),避免因节点或 DB 时钟微小差异造成重复触发 - const toleranceMs = parseInt(process.env.SCHEDULER_DEDUPE_TOLERANCE_MS || '5000', 10); - if (!Number.isNaN(lastCreatedAt) && lastCreatedAt >= (windowStartMs - toleranceMs)) { - this.recentScheduledWindowByTaskId.set(taskId, windowStartMs); - logger.info(`Task ${taskId} duplicate detected by DB guard`, { - event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, - taskId, - windowStart: new Date(windowStartMs).toISOString(), - lastExecutionCreatedAt: new Date(lastCreatedAt).toISOString(), - toleranceMs, - reason: 'db_guard', - }, LOG_CONTEXTS.EXECUTION); - return { duplicated: true, windowStart, reason: 'db_guard' }; - } - } - - this.recentScheduledWindowByTaskId.set(taskId, windowStartMs); - return { duplicated: false, windowStart }; - } - /** * 实际执行任务:创建运行记录 + 触发 Jenkins */ - private async executeTask(taskId: number, triggerReason: string, operatorId?: number): Promise { + private async executeTask(taskId: number, triggerReason: string, operatorId?: number, scheduledFor?: Date): Promise { // 从 DB 重新读取任务配置(确保最新) const row = await queryOne<{ id: number; name: string; case_ids: string; project_id: number; @@ -948,7 +601,12 @@ export class TaskSchedulerService { try { caseIds = JSON.parse(row.case_ids || '[]'); } catch { /* ignore */ } if (triggerReason === 'scheduled') { - const dedupe = await this.isDuplicateScheduledWindow(taskId, row.cron_expression); + const dedupe = await isDuplicateScheduledWindow({ + taskId, + cronExpression: row.cron_expression, + scheduledFor, + recentScheduledWindowsByTaskId: this.recentScheduledWindowsByTaskId, + }); if (dedupe.duplicated) { logger.warn(`Task ${taskId} duplicate scheduled trigger skipped`, { event: LOG_EVENTS.SCHEDULER_TASK_DUPLICATE_SKIPPED, @@ -958,12 +616,14 @@ export class TaskSchedulerService { traceId, dedupeReason: dedupe.reason, windowStart: dedupe.windowStart?.toISOString() ?? null, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); await this.recordAuditLog(taskId, 'duplicate_scheduled_skipped', SCHEDULER_USER_ID, { triggerReason, traceId, dedupeReason: dedupe.reason, windowStart: dedupe.windowStart?.toISOString() ?? null, + scheduledFor: scheduledFor?.toISOString() ?? null, }); return; } @@ -990,13 +650,14 @@ export class TaskSchedulerService { caseIds, triggerReason, traceId, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); // ⚠️ Bug Fix: 即使跳过执行,也必须更新内存中的 lastRunAt // 否则 compensateMissedFires 每次都会误判为"漏触发"并额外再 dispatch 一次, // 导致每个 cron 周期产生 2 条记录(补偿 + 正常),记录不断堆积 const cachedTask = this.taskCache.get(taskId); if (cachedTask) { - cachedTask.lastRunAt = new Date(); + cachedTask.lastRunAt = scheduledFor ?? new Date(); } return; } @@ -1012,7 +673,7 @@ export class TaskSchedulerService { status: row.status as 'active', maxRetries: row.max_retries ?? 1, retryDelayMs: row.retry_delay_ms ?? 30_000, - lastRunAt: new Date(), + lastRunAt: scheduledFor ?? new Date(), }; this.taskCache.set(taskId, updatedCache); @@ -1024,6 +685,7 @@ export class TaskSchedulerService { taskId, triggerReason, traceId, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); await this.recordAuditLog(taskId, 'skipped', SCHEDULER_USER_ID, { @@ -1031,6 +693,7 @@ export class TaskSchedulerService { triggerReason, traceId, caseCount: caseIds.length, + scheduledFor: scheduledFor?.toISOString() ?? null, }); return; @@ -1072,6 +735,7 @@ export class TaskSchedulerService { triggerReason, traceId, cronExpression: row.cron_expression, + scheduledFor: scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); // 记录审计日志:手动触发时使用实际操作者 ID,调度/重试时使用系统用户 @@ -1085,6 +749,7 @@ export class TaskSchedulerService { caseCount: caseIds.length, traceId, cronExpression: row.cron_expression, + scheduledFor: scheduledFor?.toISOString() ?? null, source: triggerReason === 'manual' ? 'manual_api' : triggerReason === 'retry' ? 'retry_timer' : 'scheduler', }); @@ -1409,9 +1074,10 @@ export class TaskSchedulerService { taskId: next.taskId, triggerReason: next.triggerReason, queueDepth: this.waitQueue.length, + scheduledFor: next.scheduledFor?.toISOString() ?? null, }, LOG_CONTEXTS.EXECUTION); - this.dispatchTask(next.taskId, next.triggerReason, next.operatorId).catch(err => { + this.dispatchTask(next.taskId, next.triggerReason, next.operatorId, next.scheduledFor).catch(err => { logger.errorLog(err, `Queue drain dispatch failed for task ${next.taskId}`, { event: LOG_EVENTS.SCHEDULER_TASK_EXECUTION_FAILED }); }); } diff --git a/server/services/TaskSchedulerService/status.ts b/server/services/TaskSchedulerService/status.ts new file mode 100644 index 0000000..7c43ea5 --- /dev/null +++ b/server/services/TaskSchedulerService/status.ts @@ -0,0 +1,63 @@ +import { CONCURRENCY_LIMIT, MAX_QUEUE_DEPTH } from './config'; +import type { DirectQueueItem, QueueItem, RunningSlot, ScheduledTask } from './types'; + +export interface TaskSchedulerStatusSnapshot { + running: Array<{ taskId: number; runId: number; elapsedMs: number; label?: string }>; + queued: Array<{ + taskId: number; + triggerReason: string; + waitMs: number; + priority: number; + queuePosition: number; + scheduledFor?: string; + }>; + directQueued: Array<{ label: string; waitMs: number; queuePosition: number }>; + scheduled: number[]; + concurrencyLimit: number; + queueDepth: number; + directQueueDepth: number; + maxQueueDepth: number; +} + +interface BuildSchedulerStatusInput { + runningSlots: Map; + waitQueue: QueueItem[]; + directQueue: DirectQueueItem[]; + taskCache: Map; +} + +export function buildSchedulerStatus({ + runningSlots, + waitQueue, + directQueue, + taskCache, +}: BuildSchedulerStatusInput): TaskSchedulerStatusSnapshot { + const now = Date.now(); + + return { + running: Array.from(runningSlots.values()).map(slot => ({ + taskId: slot.taskId, + runId: slot.runId, + elapsedMs: now - slot.startedAt, + label: slot.label, + })), + queued: waitQueue.map((item, idx) => ({ + taskId: item.taskId, + triggerReason: item.triggerReason, + waitMs: now - item.enqueuedAt, + priority: item.priority, + queuePosition: idx + 1, + scheduledFor: item.scheduledFor?.toISOString(), + })), + directQueued: directQueue.map((item, idx) => ({ + label: item.label, + waitMs: now - item.enqueuedAt, + queuePosition: idx + 1, + })), + scheduled: Array.from(taskCache.keys()), + concurrencyLimit: CONCURRENCY_LIMIT, + queueDepth: waitQueue.length, + directQueueDepth: directQueue.length, + maxQueueDepth: MAX_QUEUE_DEPTH, + }; +} diff --git a/server/services/TaskSchedulerService/types.ts b/server/services/TaskSchedulerService/types.ts index 17700df..f7d916b 100644 --- a/server/services/TaskSchedulerService/types.ts +++ b/server/services/TaskSchedulerService/types.ts @@ -15,6 +15,7 @@ export interface QueueItem { taskId: number; triggerReason: 'scheduled' | 'manual' | 'retry'; operatorId?: number; + scheduledFor?: Date; enqueuedAt: number; priority: number; timeoutTimer?: NodeJS.Timeout; diff --git a/src/hooks/useTasks.ts b/src/hooks/useTasks.ts index e09f547..58749be 100644 --- a/src/hooks/useTasks.ts +++ b/src/hooks/useTasks.ts @@ -369,6 +369,7 @@ export interface QueuedItemInfo { waitMs: number; priority: number; queuePosition: number; + scheduledFor?: string; } /** 直连等待队列项详情(run-case / run-batch 专用) */ diff --git a/src/pages/cases/AICaseCreate.tsx b/src/pages/cases/AICaseCreate.tsx index 6e6edc9..511277a 100644 --- a/src/pages/cases/AICaseCreate.tsx +++ b/src/pages/cases/AICaseCreate.tsx @@ -4,7 +4,6 @@ import { ArrowRight, ArrowUpDown, BookOpen, - Bot, Boxes, Bug, ChevronRight, @@ -14,8 +13,6 @@ import { Filter, FolderUp, Inbox, - Link2, - Loader2, NotebookTabs, PencilLine, Plus, @@ -26,14 +23,6 @@ import { import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; -import { Textarea } from '@/components/ui/textarea'; -import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, -} from '@/components/ui/sheet'; import { listAllWorkspaceDocuments } from '@/lib/aiCaseStorage'; import { computeProgress } from '@/lib/aiCaseMindMap'; import type { @@ -44,14 +33,13 @@ import type { AiCaseWorkspaceDocument, } from '@/types/aiCases'; import { useAiGeneration } from '@/contexts/AiGenerationContext'; +import { NewRequirementSheet } from './components/NewRequirementSheet'; + -function generateWorkspaceId(): string { - return `ai-ws-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} type HistoryFilterMode = 'all' | 'synced' | 'local-only'; type HistorySortMode = 'updatedAt' | 'createdAt'; -type WorkspaceInputMode = 'direct' | 'upload' | 'template'; + interface WorkspaceMetrics { progress: AiCaseProgress; @@ -74,18 +62,7 @@ interface QuickStartCardItem { onClick: () => void; } -interface InputModeOption { - id: WorkspaceInputMode; - label: string; - icon: typeof PencilLine; -} -interface SupplementOption { - id: string; - label: string; - icon: typeof FileText; - accent: string; -} const MATERIAL_CHIPS: MaterialChip[] = [ { @@ -110,42 +87,7 @@ const MATERIAL_CHIPS: MaterialChip[] = [ }, ]; -const WORKSPACE_MODULE_OPTIONS = [ - '登录与认证', - '接口联调', - '订单流程', - '支付结算', - '数据报表', - '测试平台配置', -] as const; -const INPUT_MODE_OPTIONS: InputModeOption[] = [ - { id: 'direct', label: '直接输入', icon: PencilLine }, - { id: 'upload', label: '上传文档', icon: FolderUp }, - { id: 'template', label: '从模板开始', icon: Boxes }, -]; - -const SUPPLEMENT_OPTIONS: SupplementOption[] = [ - { id: 'prd', label: 'PRD / 需求文档', icon: FileText, accent: 'text-violet-600' }, - { id: 'openapi', label: 'OpenAPI / 接口文档', icon: NotebookTabs, accent: 'text-emerald-600' }, - { id: 'attachment', label: '附件上传', icon: Link2, accent: 'text-blue-600' }, - { id: 'code', label: '代码变更', icon: Code2, accent: 'text-amber-600' }, - { id: 'bug', label: '缺陷单', icon: Bug, accent: 'text-rose-500' }, -]; - -const NEXT_STEP_ITEMS = [ - '补充输入材料', - '生成结构化测试用例', - '查看覆盖与风险', - '执行与回流', -] as const; - -const EXAMPLE_REQUIREMENT_TEXT = [ - '业务目标:优化登录模块测试设计,覆盖账号密码登录、验证码、记住我和登录失败限制。', - '核心流程:用户输入邮箱和密码后提交;支持忘记密码;连续输错 5 次后账户临时锁定。', - '边界条件:空值、超长输入、错误密码、过期验证码、已锁定账号。', - '异常场景:服务超时、接口 500、登录态失效、频繁重试。', -].join('\n'); const HISTORY_FILTER_LABELS: Record = { all: '全部', @@ -358,283 +300,7 @@ function HeroBackdrop() { ); } -interface NewRequirementSheetProps { - open: boolean; - onOpenChange: (open: boolean) => void; -} - -function NewRequirementSheet({ open, onOpenChange }: NewRequirementSheetProps) { - const [, setLocation] = useLocation(); - const { notifyStart } = useAiGeneration(); - const [workspaceName, setWorkspaceName] = useState(''); - const [workspaceModule, setWorkspaceModule] = useState(''); - const [requirementText, setRequirementText] = useState(''); - const [inputMode, setInputMode] = useState('direct'); - const [selectedSupplements, setSelectedSupplements] = useState(['prd', 'openapi']); - const [isSubmitting, setIsSubmitting] = useState(false); - - const resetForm = () => { - setWorkspaceName(''); - setWorkspaceModule(''); - setRequirementText(''); - setInputMode('direct'); - setSelectedSupplements(['prd', 'openapi']); - }; - - const navigateToWorkspace = async (autoGenerate: boolean) => { - if (!requirementText.trim()) { - toast.error('请先输入需求描述'); - return; - } - - setIsSubmitting(true); - try { - const docId = generateWorkspaceId(); - notifyStart(docId); - - onOpenChange(false); - resetForm(); - - const params = new URLSearchParams(); - params.set('docId', docId); - params.set('autoGenerate', autoGenerate ? 'true' : 'false'); - params.set('initName', workspaceName.trim() || 'AI Testcase Workspace'); - params.set('initReq', requirementText.trim()); - setLocation(`/cases/ai?${params.toString()}`); - } catch (error) { - console.error('[AICaseCreate] failed to navigate', error); - toast.error('跳转失败,请重试'); - } finally { - setIsSubmitting(false); - } - }; - - const handleGenerate = async () => { - await navigateToWorkspace(true); - }; - - const handleCreateOnly = async () => { - await navigateToWorkspace(false); - }; - - const handleClose = () => { - if (!isSubmitting) { - onOpenChange(false); - resetForm(); - } - }; - - const toggleSupplement = (id: string) => { - setSelectedSupplements((current) => - current.includes(id) ? current.filter((item) => item !== id) : [...current, id] - ); - }; - - return ( - - - -
-
- -
-
- - 新增 AI 工作台 - - - 输入需求背景或导入材料,创建一个可继续补充、生成、执行与回流的 AI 工作台。 - -
-
-
- -
-
-
基础信息
- -
- - setWorkspaceName(event.target.value)} - placeholder="例如:登录模块测试设计" - className="h-11 rounded-xl border-slate-200 bg-white" - /> -

- 建议填写清晰的业务模块或场景名称,便于后续继续协作。 -

-
- -
- - -
-
- -
-
输入方式
-
- {INPUT_MODE_OPTIONS.map((option) => { - const Icon = option.icon; - const active = inputMode === option.id; - return ( - - ); - })} -
-

- 你也可以先创建工作台,再到侧边补充资料或切换更多来源。 -

-
- -
-
- - -
-