diff --git a/README.md b/README.md index 8e3d6f2..1bec52f 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,20 @@ QAS_URL=https://qas.eu1.qasphere.com ## Commands: `junit-upload`, `playwright-json-upload` -The `junit-upload` and `playwright-json-upload` commands upload test results from JUnit XML and Playwright JSON reports to QA Sphere respectively. Both commands can either create a new test run within a QA Sphere project or upload results to an existing run, and they share the same set of options. +The `junit-upload` and `playwright-json-upload` commands upload test results from JUnit XML and Playwright JSON reports to QA Sphere respectively. + +There are two modes for uploading results using the commands: + +1. Upload to an existing test run by specifying its URL via `--run-url` flag +2. Create a new test run and upload results to it (when `--run-url` flag is not specified) ### Options -- `-r, --run-url` - Optional URL of an existing run for uploading results (a new run is created if not specified) -- `--run-name` - Optional name template for creating new test run when run url is not specified (supports `{env:VAR}`, `{YYYY}`, `{YY}`, `{MM}`, `{MMM}`, `{DD}`, `{HH}`, `{hh}`, `{mm}`, `{ss}`, `{AMPM}` placeholders). If not specified, `Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}` is used as default +- `-r`/`--run-url` - Upload results to an existing test run +- `--project-code`, `--run-name`, `--create-tcases` - Create a new test run and upload results to it + - `--project-code` - Project code for creating new test run. It can also be auto detected from test case markers in the results, but this is not fully reliable, so it is recommended to specify the project code explicitly + - `--run-name` - Optional name template for creating new test run. It supports `{env:VAR}`, `{YYYY}`, `{YY}`, `{MM}`, `{MMM}`, `{DD}`, `{HH}`, `{hh}`, `{mm}`, `{ss}`, `{AMPM}` placeholders (default: `Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}`) + - `--create-tcases` - Automatically create test cases in QA Sphere for results that don't have valid test case markers. A mapping file (`qasphere-automapping-YYYYMMDD-HHmmss.txt`) is generated showing the sequence numbers assigned to each new test case (default: `false`) - `--attachments` - Try to detect and upload any attachments with the test result - `--force` - Ignore API request errors, invalid test cases, or attachments - `--ignore-unmatched` - Suppress individual unmatched test messages, show summary only @@ -97,39 +105,41 @@ Ensure the required environment variables are defined before running these comma **Note:** The following examples use `junit-upload`, but you can replace it with `playwright-json-upload` and adjust the file extension from `.xml` to `.json` to upload Playwright JSON reports instead. -1. Create a new test run with default name template (`Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}`) and upload results: +1. Upload to an existing test run: ```bash - qasphere junit-upload ./test-results.xml + qasphere junit-upload -r https://qas.eu1.qasphere.com/project/P1/run/23 ./test-results.xml ``` -2. Upload to an existing test run: +2. Create a new test run with default name template and upload results: ```bash - qasphere junit-upload -r https://qas.eu1.qasphere.com/project/P1/run/23 ./test-results.xml + qasphere junit-upload ./test-results.xml ``` + Project code is detected from test case markers in the results. + 3. Create a new test run with name template without any placeholders and upload results: ```bash - qasphere junit-upload --run-name "v1.4.4-rc5" ./test-results.xml + qasphere junit-upload --project-code P1 --run-name "v1.4.4-rc5" ./test-results.xml ``` 4. Create a new test run with name template using environment variables and date placeholders and upload results: ```bash - qasphere junit-upload --run-name "CI Build {env:BUILD_NUMBER} - {YYYY}-{MM}-{DD}" ./test-results.xml + qasphere junit-upload --project-code P1 --run-name "CI Build {env:BUILD_NUMBER} - {YYYY}-{MM}-{DD}" ./test-results.xml ``` If `BUILD_NUMBER` environment variable is set to `v1.4.4-rc5` and today's date is January 1, 2025, the run would be named "CI Build v1.4.4-rc5 - 2025-01-01". -5. Create a new test run with name template using date/time placeholders and upload results: +5. Create a new test run with name template using date/time placeholders and create test cases for results without valid markers and upload results: ```bash - qasphere junit-upload --run-name "Nightly Tests {YYYY}/{MM}/{DD} {HH}:{mm}" ./test-results.xml + qasphere junit-upload --project-code P1 --run-name "Nightly Tests {YYYY}/{MM}/{DD} {HH}:{mm}" --create-tcases ./test-results.xml ``` - If the current time is 10:34 PM on January 1, 2025, the run would be named "Nightly Tests 2025/01/01 22:34". + If the current time is 10:34 PM on January 1, 2025, the run would be named "Nightly Tests 2025/01/01 22:34". This also creates new test cases in QA Sphere for any results that doesn't have a valid test case marker. A mapping file (`qasphere-automapping-YYYYMMDD-HHmmss.txt`) is generated showing the sequence numbers assigned to each newly created test case. Update your test cases to include the markers in the name, for future uploads. 6. Upload results with attachments: @@ -151,7 +161,7 @@ Ensure the required environment variables are defined before running these comma This will show only a summary like "Skipped 5 unmatched tests" instead of individual error messages for each unmatched test. -9. Skip stdout/stderr for passed tests to reduce result payload size: +9. Skip stdout for passed tests to reduce result payload size: ```bash qasphere junit-upload --skip-report-stdout on-success ./test-results.xml @@ -159,17 +169,15 @@ Ensure the required environment variables are defined before running these comma This will exclude stdout from passed tests while still including it for failed, blocked, or skipped tests. - Skip both stdout and stderr for passed tests: - - ```bash - qasphere junit-upload --skip-report-stdout on-success --skip-report-stderr on-success ./test-results.xml - ``` - - This is useful when you have verbose logging in tests but only want to see output for failures. +10. Skip both stdout and stderr for passed tests: + ```bash + qasphere junit-upload --skip-report-stdout on-success --skip-report-stderr on-success ./test-results.xml + ``` + This is useful when you have verbose logging in tests but only want to see output for failures. ## Test Report Requirements -The QAS CLI requires test cases in your reports (JUnit XML or Playwright JSON) to reference corresponding test cases in QA Sphere. These references are used to map test results from your automation to the appropriate test cases in QA Sphere. If a report lacks these references or the referenced test case doesn't exist in QA Sphere, the tool will display an error message. +The QAS CLI maps test results from your reports (JUnit XML or Playwright JSON) to corresponding test cases in QA Sphere using test case markers. If a test result lacks a valid marker, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results. ### JUnit XML diff --git a/src/api/folders.ts b/src/api/folders.ts new file mode 100644 index 0000000..964dc54 --- /dev/null +++ b/src/api/folders.ts @@ -0,0 +1,14 @@ +import { Folder, GetFoldersRequest, PaginatedResponse, ResourceId } from './schemas' +import { appendSearchParams, jsonResponse, withJson } from './utils' + +export const createFolderApi = (fetcher: typeof fetch) => { + fetcher = withJson(fetcher) + return { + getFoldersPaginated: (projectCode: ResourceId, request: GetFoldersRequest) => + fetcher( + appendSearchParams(`/api/public/v0/project/${projectCode}/tcase/folders`, request) + ).then((r) => jsonResponse>(r)), + } +} + +export type FolderApi = ReturnType diff --git a/src/api/index.ts b/src/api/index.ts index 5e22392..f9cd166 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,15 +1,17 @@ +import { createFileApi } from './file' +import { createFolderApi } from './folders' import { createProjectApi } from './projects' import { createRunApi } from './run' import { createTCaseApi } from './tcases' -import { createFileApi } from './file' import { withApiKey, withBaseUrl } from './utils' const getApi = (fetcher: typeof fetch) => { return { + files: createFileApi(fetcher), + folders: createFolderApi(fetcher), projects: createProjectApi(fetcher), runs: createRunApi(fetcher), testcases: createTCaseApi(fetcher), - file: createFileApi(fetcher), } } diff --git a/src/api/schemas.ts b/src/api/schemas.ts index 16e809c..61b0425 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -2,8 +2,55 @@ export type ResourceId = string | number export type ResultStatus = 'open' | 'passed' | 'blocked' | 'failed' | 'skipped' +export interface PaginatedResponse { + data: T[] + total: number + page: number + limit: number +} + +export interface PaginatedRequest { + page?: number + limit?: number +} + +export interface TCase { + id: string + legacyId?: string + seq: number + title: string + version: number + projectId: string + folderId: number +} + +export interface CreateTCasesRequest { + folderPath: string[] + tcases: { title: string; tags: string[] }[] +} + +export interface CreateTCasesResponse { + tcases: { id: string; seq: number }[] +} + +export interface GetTCasesRequest extends PaginatedRequest { + folders?: number[] +} + +export interface GetTCasesBySeqRequest { + seqIds: string[] + page?: number + limit?: number +} + +export interface GetFoldersRequest extends PaginatedRequest { + search?: string +} + export interface Folder { id: number + parentId: number + pos: number title: string } diff --git a/src/api/tcases.ts b/src/api/tcases.ts index 44843e7..8c758c3 100644 --- a/src/api/tcases.ts +++ b/src/api/tcases.ts @@ -1,34 +1,32 @@ -import { ResourceId } from './schemas' -import { jsonResponse, withJson } from './utils' -export interface PaginatedResponse { - data: T[] - total: number - page: number - limit: number -} - -export interface TCaseBySeq { - id: string - legacyId?: string - seq: number - version: number - projectId: string - folderId: number -} - -export interface GetTCasesBySeqRequest { - seqIds: string[] - page?: number - limit?: number -} +import { + CreateTCasesRequest, + CreateTCasesResponse, + GetTCasesBySeqRequest, + GetTCasesRequest, + PaginatedResponse, + ResourceId, + TCase, +} from './schemas' +import { appendSearchParams, jsonResponse, withJson } from './utils' export const createTCaseApi = (fetcher: typeof fetch) => { fetcher = withJson(fetcher) return { + getTCasesPaginated: (projectCode: ResourceId, request: GetTCasesRequest) => + fetcher(appendSearchParams(`/api/public/v0/project/${projectCode}/tcase`, request)).then( + (r) => jsonResponse>(r) + ), + getTCasesBySeq: (projectCode: ResourceId, request: GetTCasesBySeqRequest) => fetcher(`/api/public/v0/project/${projectCode}/tcase/seq`, { method: 'POST', body: JSON.stringify(request), - }).then((r) => jsonResponse>(r)), + }).then((r) => jsonResponse>(r)), + + createTCases: (projectCode: ResourceId, request: CreateTCasesRequest) => + fetcher(`/api/public/v0/project/${projectCode}/tcase/bulk`, { + method: 'POST', + body: JSON.stringify(request), + }).then((r) => jsonResponse(r)), } } diff --git a/src/api/utils.ts b/src/api/utils.ts index 4f459db..9a58649 100644 --- a/src/api/utils.ts +++ b/src/api/utils.ts @@ -48,3 +48,39 @@ export const jsonResponse = async (response: Response): Promise => { } throw new Error(response.statusText) } + +const updateSearchParams = (searchParams: URLSearchParams, obj?: T) => { + const isValidValue = (value: unknown) => { + return value !== undefined && value !== null + } + + if (!obj) return + + Object.entries(obj).forEach(([key, value]) => { + if (isValidValue(value)) { + if (Array.isArray(value)) { + value.forEach((param) => { + if (isValidValue(param)) { + searchParams.append(key, String(param)) + } + }) + } else if (value instanceof Date) { + searchParams.set(key, value.toISOString()) + } else if (typeof value === 'object') { + updateSearchParams(searchParams, value) + } else { + searchParams.set(key, String(value)) + } + } + }) +} + +export const appendSearchParams = (pathname: string, obj: T): string => { + const searchParams = new URLSearchParams() + updateSearchParams(searchParams, obj) + + if (searchParams.size > 0) { + return `${pathname}?${searchParams.toString()}` + } + return pathname +} diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index af76ca9..78b6687 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -1,6 +1,6 @@ import { Arguments, Argv, CommandModule } from 'yargs' import chalk from 'chalk' -import { loadEnvs } from '../utils/env' +import { loadEnvs, qasEnvFile } from '../utils/env' import { ResultUploadCommandArgs, ResultUploadCommandHandler, @@ -36,11 +36,22 @@ export class ResultUploadCommandModule implements CommandModule + + + + + + + + + + + + diff --git a/src/tests/fixtures/playwright-json/without-markers.json b/src/tests/fixtures/playwright-json/without-markers.json new file mode 100644 index 0000000..3cf230b --- /dev/null +++ b/src/tests/fixtures/playwright-json/without-markers.json @@ -0,0 +1,79 @@ +{ + "suites": [ + { + "title": "ui.cart.spec.ts", + "specs": [ + { + "title": "Test cart TEST-002", + "tags": [], + "tests": [ + { + "annotations": [], + "expectedStatus": "passed", + "projectName": "chromium", + "results": [ + { + "status": "passed", + "errors": [], + "stdout": [], + "stderr": [], + "retry": 0, + "duration": 1000, + "attachments": [] + } + ], + "status": "expected" + } + ] + }, + { + "title": "The cart is still filled after refreshing the page", + "tags": [], + "tests": [ + { + "annotations": [], + "expectedStatus": "passed", + "projectName": "chromium", + "results": [ + { + "status": "passed", + "errors": [], + "stdout": [], + "stderr": [], + "retry": 0, + "duration": 1000, + "attachments": [] + } + ], + "status": "expected" + } + ] + }, + { + "title": "TEST-010: Cart should be cleared after making the checkout", + "tags": [], + "tests": [ + { + "annotations": [], + "expectedStatus": "passed", + "projectName": "chromium", + "results": [ + { + "status": "passed", + "errors": [], + "stdout": [], + "stderr": [], + "retry": 0, + "duration": 1000, + "attachments": [] + } + ], + "status": "expected" + } + ] + } + ], + "suites": [] + } + ] +} diff --git a/src/tests/result-upload.spec.ts b/src/tests/result-upload.spec.ts index fa2e1fb..3aee549 100644 --- a/src/tests/result-upload.spec.ts +++ b/src/tests/result-upload.spec.ts @@ -1,10 +1,19 @@ -import { afterAll, beforeAll, expect, test, describe, afterEach } from 'vitest' -import { run } from '../commands/main' -import { setupServer } from 'msw/node' import { HttpResponse, http } from 'msw' +import { setupServer } from 'msw/node' +import { unlinkSync, readdirSync } from 'node:fs' +import { afterAll, beforeAll, beforeEach, expect, test, describe, afterEach } from 'vitest' +import { run } from '../commands/main' +import { + CreateTCasesRequest, + CreateTCasesResponse, + Folder, + PaginatedResponse, + TCase, +} from '../api/schemas' +import { DEFAULT_FOLDER_TITLE } from '../utils/result-upload/ResultUploadCommandHandler' +import { setMaxResultsInRequest } from '../utils/result-upload/ResultUploader' import { runTestCases } from './fixtures/testcases' import { countMockedApiCalls } from './utils' -import { setMaxResultsInRequest } from '../utils/result-upload/ResultUploader' const projectCode = 'TEST' const runId = '1' @@ -15,14 +24,29 @@ const runURL = `${baseURL}/project/${projectCode}/run/${runId}` process.env['QAS_TOKEN'] = 'QAS_TOKEN' process.env['QAS_URL'] = baseURL -let lastCreatedRunTitle = '' -let createRunTitleConflict = false +let lastCreatedRunTitle = '' // Stores title in the request, for the last create run API call +let createRunTitleConflict = false // If true, the create run API returns a title conflict error +let createTCasesResponse: CreateTCasesResponse | null = null // Stores mock response for the create tcases API call +let overriddenGetPaginatedTCasesResponse: PaginatedResponse | null = null // Stores overridden (non-default) response for the get tcases API call +let overriddenGetFoldersResponse: PaginatedResponse | null = null // Stores overridden (non-default) response for the get folders API call const server = setupServer( http.get(`${baseURL}/api/public/v0/project/${projectCode}`, ({ request }) => { expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') return HttpResponse.json({ exists: true }) }), + http.get(`${baseURL}/api/public/v0/project/${projectCode}/tcase/folders`, ({ request }) => { + expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') + return HttpResponse.json( + overriddenGetFoldersResponse || { data: [], total: 0, page: 1, limit: 50 } + ) + }), + http.get(`${baseURL}/api/public/v0/project/${projectCode}/tcase`, ({ request }) => { + expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') + return HttpResponse.json( + overriddenGetPaginatedTCasesResponse || { data: [], total: 0, page: 1, limit: 50 } + ) + }), http.post(`${baseURL}/api/public/v0/project/${projectCode}/tcase/seq`, ({ request }) => { expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') return HttpResponse.json({ @@ -30,6 +54,33 @@ const server = setupServer( total: runTestCases.length, }) }), + http.post(`${baseURL}/api/public/v0/project/${projectCode}/tcase/bulk`, async ({ request }) => { + expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') + + if (!createTCasesResponse) { + return HttpResponse.json( + { + message: 'No mock response set for create tcases API call', + }, + { + status: 500, + } + ) + } + + const body = (await request.json()) as CreateTCasesRequest + if (body.tcases.length !== createTCasesResponse.tcases.length) { + return HttpResponse.json( + { + message: `${body.tcases.length} test cases in request does not match ${createTCasesResponse.tcases.length} in the mock response`, + }, + { + status: 400, + } + ) + } + return HttpResponse.json(createTCasesResponse) + }), http.post(`${baseURL}/api/public/v0/project/${projectCode}/run`, async ({ request }) => { expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') const body = (await request.json()) as { title: string } @@ -91,6 +142,22 @@ const countFileUploadApiCalls = () => countMockedApiCalls(server, (req) => req.url.endsWith('/file')) const countResultUploadApiCalls = () => countMockedApiCalls(server, (req) => new URL(req.url).pathname.endsWith('/result/batch')) +const countCreateTCasesApiCalls = () => + countMockedApiCalls(server, (req) => new URL(req.url).pathname.endsWith('/tcase/bulk')) + +const getMappingFiles = () => + new Set( + readdirSync('.').filter((f) => f.startsWith('qasphere-automapping-') && f.endsWith('.txt')) + ) + +const cleanupGeneratedMappingFiles = (existingMappingFiles?: Set) => { + const currentFiles = getMappingFiles() + currentFiles.forEach((f) => { + if (!existingMappingFiles?.has(f)) { + unlinkSync(f) + } + }) +} const fileTypes = [ { @@ -160,7 +227,7 @@ fileTypes.forEach((fileType) => { }) }) - describe('Uploading test results', () => { + describe('Uploading test results with run URL', () => { test('Test cases on reports with all matching test cases on QAS should be successful', async () => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() @@ -351,5 +418,89 @@ fileTypes.forEach((fileType) => { ) }) }) + + describe('Uploading test results with test case creation', () => { + let existingMappingFiles: Set | undefined = undefined + + beforeEach(() => { + existingMappingFiles = getMappingFiles() + }) + + afterEach(() => { + cleanupGeneratedMappingFiles(existingMappingFiles) + existingMappingFiles = undefined + createTCasesResponse = null + overriddenGetPaginatedTCasesResponse = null + overriddenGetFoldersResponse = null + }) + + test('Should create new test cases for results without valid markers', async () => { + const numCreateTCasesCalls = countCreateTCasesApiCalls() + const numResultUploadCalls = countResultUploadApiCalls() + + setMaxResultsInRequest(1) + createTCasesResponse = { + tcases: [ + { id: '6', seq: 6 }, + { id: '7', seq: 7 }, + ], + } + + await run( + `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + ) + expect(numCreateTCasesCalls()).toBe(1) + expect(numResultUploadCalls()).toBe(3) // 3 results total + }) + + test('Should not create new test case if one with same title already exists', async () => { + const numCreateTCasesCalls = countCreateTCasesApiCalls() + const numResultUploadCalls = countResultUploadApiCalls() + + setMaxResultsInRequest(1) + overriddenGetFoldersResponse = { + data: [{ id: 1, title: DEFAULT_FOLDER_TITLE, parentId: 0, pos: 0 }], + total: 1, + page: 1, + limit: 50, + } + overriddenGetPaginatedTCasesResponse = { + data: [ + { + id: '6', + seq: 6, + title: 'The cart is still filled after refreshing the page', + version: 1, + projectId: 'projectid', + folderId: 1, + }, + ], + total: 1, + page: 1, + limit: 50, + } + createTCasesResponse = { + tcases: [{ id: '7', seq: 7 }], + } + + await run( + `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + ) + expect(numCreateTCasesCalls()).toBe(1) + expect(numResultUploadCalls()).toBe(3) // 3 results total + }) + + test('Should not create new test cases if all results have valid markers', async () => { + const numCreateTCasesCalls = countCreateTCasesApiCalls() + const numResultUploadCalls = countResultUploadApiCalls() + + setMaxResultsInRequest(1) + await run( + `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + ) + expect(numCreateTCasesCalls()).toBe(0) + expect(numResultUploadCalls()).toBe(5) // 5 results total + }) + }) }) }) diff --git a/src/utils/misc.ts b/src/utils/misc.ts index dd70186..2f411ad 100644 --- a/src/utils/misc.ts +++ b/src/utils/misc.ts @@ -96,6 +96,10 @@ export const parseTCaseUrl = (url: string) => { } } +export const getTCaseMarker = (projectCode: string, seq: number) => { + return `${projectCode}-${seq.toString().padStart(3, '0')}` +} + export const printErrorThenExit = (e: unknown): never => { printError(e) process.exit(1) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index a62433a..f4b90be 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -1,10 +1,10 @@ import { Arguments } from 'yargs' import chalk from 'chalk' -import { readFileSync } from 'node:fs' +import { readFileSync, writeFileSync } from 'node:fs' import { dirname } from 'node:path' -import { parseRunUrl, printErrorThenExit, processTemplate } from '../misc' +import { getTCaseMarker, parseRunUrl, printErrorThenExit, processTemplate } from '../misc' import { Api, createApi } from '../../api' -import { PaginatedResponse, TCaseBySeq } from '../../api/tcases' +import { TCase } from '../../api/schemas' import { TestCaseResult } from './types' import { ResultUploader } from './ResultUploader' import { parseJUnitXml } from './junitXmlParser' @@ -25,23 +25,40 @@ export type Parser = ( options: ParserOptions ) => Promise -export interface ResultUploadCommandArgs { +export type ResultUploadCommandArgs = { type: UploadCommandType - runUrl?: string - runName?: string files: string[] force: boolean attachments: boolean ignoreUnmatched: boolean skipReportStdout: SkipOutputOption skipReportStderr: SkipOutputOption -} +} & ( + | { + runUrl: string + } + | { + projectCode?: string + runName?: string + createTcases: boolean + } +) interface FileResults { file: string results: TestCaseResult[] } +interface TestCaseResultWithSeqAndFile { + seq: number | null + file: string + result: TestCaseResult +} + +const DEFAULT_PAGE_SIZE = 5000 +export const DEFAULT_FOLDER_TITLE = 'cli-import' +const DEFAULT_TCASE_TAGS = ['cli-import'] +const DEFAULT_MAPPING_FILENAME_TEMPLATE = 'qasphere-automapping-{YYYY}{MM}{DD}-{HH}{mm}{ss}.txt' const commandTypeParsers: Record = { 'junit-upload': parseJUnitXml, 'playwright-json-upload': parsePlaywrightJson, @@ -66,12 +83,11 @@ export class ResultUploadCommandHandler { return printErrorThenExit('No files specified') } - const fileResults = await this.parseFiles() - const results = fileResults.flatMap((fileResult) => fileResult.results) - + let fileResults = await this.parseFiles() let projectCode = '' let runId = 0 - if (this.args.runUrl) { + + if ('runUrl' in this.args && this.args.runUrl) { // Handle existing run URL console.log(chalk.blue(`Using existing test run: ${this.args.runUrl}`)) @@ -85,22 +101,25 @@ export class ResultUploadCommandHandler { runId = urlParsed.run projectCode = urlParsed.project } else { - // Auto-detect project from results - projectCode = this.detectProjectCode(results) - console.log(chalk.blue(`Detected project code: ${projectCode}`)) + if (this.args.projectCode) { + projectCode = this.args.projectCode as string + } else { + // Try to auto-detect project code from results. This is not fully reliable, but + // is kept for backward compatibility. Better to specify project code explicitly + projectCode = this.detectProjectCodeFromTCaseNames(fileResults) + console.log(chalk.blue(`Detected project code: ${projectCode}`)) + } - // Create a new test run if (!(await this.api.projects.checkProjectExists(projectCode))) { return printErrorThenExit(`Project ${projectCode} does not exist`) } - console.log(chalk.blue(`Creating a new test run for project: ${projectCode}`)) - const tcaseRefs = this.extractTestCaseRefs(projectCode, fileResults) - const tcases = await this.getTestCases(projectCode, tcaseRefs) - runId = await this.createNewRun(projectCode, tcases) - console.log(chalk.blue(`Test run URL: ${this.baseUrl}/project/${projectCode}/run/${runId}`)) + const resp = await this.getTCaseIds(projectCode, fileResults) + fileResults = resp.fileResults + runId = await this.createNewRun(projectCode, resp.tcaseIds) } + const results = fileResults.flatMap((fileResult) => fileResult.results) await this.uploadResults(projectCode, runId, results) } @@ -125,26 +144,49 @@ export class ResultUploadCommandHandler { return results } - protected detectProjectCode(results: TestCaseResult[]) { - for (const result of results) { - if (result.name) { - // Look for pattern like PRJ-123 or TEST-456 - const match = result.name.match(/([A-Za-z0-9]{1,5})-\d{3,}/) - if (match) { - return match[1] + protected detectProjectCodeFromTCaseNames(fileResults: FileResults[]) { + // Look for pattern like PRJ-123 or TEST-456 + const tcaseSeqPattern = String.raw`([A-Za-z0-9]{1,5})-\d{3,}` + for (const { results } of fileResults) { + for (const result of results) { + if (result.name) { + const match = this.execRegexWithPriority(tcaseSeqPattern, result.name) + if (match) { + return match[1] + } } } } return printErrorThenExit( - 'Could not detect project code from test case names. Please make sure they contain a valid project code (e.g., PRJ-123)' + 'Could not detect project code from test case names. Please specify project code using --project-code flag' ) } - protected extractTestCaseRefs(projectCode: string, fileResults: FileResults[]): Set { - const tcaseRefs = new Set() + private execRegexWithPriority(pattern: string, str: string): RegExpExecArray | null { + // Try matching at start first + const startRegex = new RegExp(`^${pattern}`) + let match = startRegex.exec(str) + if (match) return match + + // Try matching at end + const endRegex = new RegExp(`${pattern}$`) + match = endRegex.exec(str) + if (match) return match + + // Fall back to matching anywhere + const anywhereRegex = new RegExp(pattern) + return anywhereRegex.exec(str) + } + + protected async getTCaseIds(projectCode: string, fileResults: FileResults[]) { const shouldFailOnInvalid = !this.args.force && !this.args.ignoreUnmatched + const tcaseSeqPattern = String.raw`${projectCode}-(\d{3,})` + const seqIdsSet: Set = new Set() + const resultsWithSeqAndFile: TestCaseResultWithSeqAndFile[] = [] + + // First extract the sequence numbers from the test case names for (const { file, results } of fileResults) { for (const result of results) { if (!result.name) { @@ -154,59 +196,225 @@ export class ResultUploadCommandHandler { continue } - const match = new RegExp(`${projectCode}-(\\d{3,})`).exec(result.name) + const match = this.execRegexWithPriority(tcaseSeqPattern, result.name) + resultsWithSeqAndFile.push({ + seq: match ? Number(match[1]) : null, + file, + result, + }) + if (match) { - tcaseRefs.add(`${projectCode}-${match[1]}`) - continue + seqIdsSet.add(Number(match[1])) } + } + } + + // Now fetch the test cases by their sequence numbers + const apiTCasesMap: Record = {} + if (seqIdsSet.size > 0) { + const tcaseMarkers = Array.from(seqIdsSet).map((v) => getTCaseMarker(projectCode, v)) + + for (let page = 1; ; page++) { + const response = await this.api.testcases.getTCasesBySeq(projectCode, { + seqIds: tcaseMarkers, + page, + limit: DEFAULT_PAGE_SIZE, + }) - if (shouldFailOnInvalid) { - return printErrorThenExit( - `Test case name "${result.name}" in ${file} does not contain valid sequence number with project code (e.g., ${projectCode}-123)` - ) + for (const tcase of response.data) { + apiTCasesMap[tcase.seq] = tcase } + + if (response.data.length < DEFAULT_PAGE_SIZE) { + break + } + } + } + + // Now validate that the test cases with found sequence numbers actually exist + const tcaseIds: string[] = [] + const tcasesToCreateMap: Record = {} + for (const { seq, file, result } of resultsWithSeqAndFile) { + if (seq && apiTCasesMap[seq]) { + tcaseIds.push(apiTCasesMap[seq].id) + continue + } + + if (this.args.createTcases) { + const tcaseResults = tcasesToCreateMap[result.name] || [] + tcaseResults.push(result) + tcasesToCreateMap[result.name] = tcaseResults + continue + } + + if (shouldFailOnInvalid) { + return printErrorThenExit( + `Test case name "${result.name}" in ${file} does not contain valid sequence number with project code (e.g., ${projectCode}-123)` + ) } } - if (tcaseRefs.size === 0) { - return printErrorThenExit('No valid test case references found in any of the files') + // Create new test cases, if same is requested + if (Object.keys(tcasesToCreateMap).length > 0) { + const keys = Object.keys(tcasesToCreateMap) + const newTCases = await this.createNewTCases(projectCode, keys) + + for (let i = 0; i < keys.length; i++) { + const marker = getTCaseMarker(projectCode, newTCases[i].seq) + for (const result of tcasesToCreateMap[keys[i]] || []) { + // Prefix the test case markers for use in ResultUploader. The fileResults array + // containing the updated name is returned to the caller + result.name = `${marker}: ${result.name}` + } + tcaseIds.push(newTCases[i].id) + } } - return tcaseRefs + if (tcaseIds.length === 0) { + return printErrorThenExit('No valid test cases found in any of the files') + } + + return { tcaseIds, fileResults } } - private async getTestCases(projectCode: string, tcaseRefs: Set) { - const response = await this.api.testcases.getTCasesBySeq(projectCode, { - seqIds: Array.from(tcaseRefs), - page: 1, - limit: tcaseRefs.size, + private async createNewTCases(projectCode: string, tcasesToCreate: string[]) { + console.log(chalk.blue(`Creating test cases for results with no test case markers`)) + + // First fetch the default folder ID where we are creating new test cases. + // Ideally, there shouldn't be the need to fetch more than one page. + let defaultFolderId = null + for (let page = 1; ; page++) { + const response = await this.api.folders.getFoldersPaginated(projectCode, { + search: DEFAULT_FOLDER_TITLE, + page, + limit: DEFAULT_PAGE_SIZE, + }) + + for (const folder of response.data) { + if (folder.title === DEFAULT_FOLDER_TITLE && !folder.parentId) { + defaultFolderId = folder.id + break + } + } + + if (defaultFolderId || response.data.length < DEFAULT_PAGE_SIZE) { + break + } + } + + // If the default folder exists, fetch the test cases in it + const apiTCasesMap: Record = {} + if (defaultFolderId) { + for (let page = 1; ; page++) { + const response = await this.api.testcases.getTCasesPaginated(projectCode, { + folders: [defaultFolderId], + page, + limit: DEFAULT_PAGE_SIZE, + }) + + for (const tcase of response.data) { + apiTCasesMap[tcase.title] = tcase + } + + if (response.data.length < DEFAULT_PAGE_SIZE) { + break + } + } + } + + // Reuse existing test cases with the same title from the default folder + const ret: { id: string; seq: number }[] = [] + const idxToFill: number[] = [] + const finalTCasesToCreate: string[] = [] + for (let i = 0; i < tcasesToCreate.length; i++) { + const existingTcase = apiTCasesMap[tcasesToCreate[i]] + if (existingTcase) { + // TCase with this title already exists, reuse it + ret.push({ id: existingTcase.id, seq: existingTcase.seq }) + continue + } + + // Add a placeholder for the new test case. Will be updated later + ret.push({ id: '', seq: 0 }) + finalTCasesToCreate.push(tcasesToCreate[i]) + idxToFill.push(i) + } + + if (!finalTCasesToCreate.length) { + console.log( + chalk.blue( + `Reusing ${ret.length} test cases with same title from "${DEFAULT_FOLDER_TITLE}" folder, no new test cases created` + ) + ) + return ret + } + + // Create new test cases and update the placeholders with the actual test case IDs + const { tcases } = await this.api.testcases.createTCases(projectCode, { + folderPath: [DEFAULT_FOLDER_TITLE], + tcases: finalTCasesToCreate.map((title) => ({ title, tags: DEFAULT_TCASE_TAGS })), }) - if (response.total === 0 || response.data.length === 0) { - return printErrorThenExit('No matching test cases found in the project') + console.log( + chalk.green( + `Created ${tcases.length} new test cases in folder "${DEFAULT_FOLDER_TITLE}"${ + ret.length > tcases.length + ? ` and reused ${ret.length - tcases.length} test cases with same title` + : '' + }` + ) + ) + + for (let i = 0; i < idxToFill.length; i++) { + ret[idxToFill[i]] = tcases[i] } - return response + try { + const mappingFilename = processTemplate(DEFAULT_MAPPING_FILENAME_TEMPLATE) + const mappingLines = tcases + .map((t, i) => `${getTCaseMarker(projectCode, t.seq)}: ${tcasesToCreate[i]}`) + .join('\n') + + writeFileSync(mappingFilename, mappingLines) + console.log( + chalk.yellow( + `Created mapping file for newly created test cases: ${mappingFilename}\nUpdate your test cases to include the test case markers in the name, for future uploads` + ) + ) + } catch (err) { + console.log( + chalk.yellow( + `Warning: Failed to write test case mapping file: ${ + err instanceof Error ? err.message : String(err) + }` + ) + ) + } + + return ret } - private async createNewRun(projectCode: string, tcases: PaginatedResponse) { + private async createNewRun(projectCode: string, tcaseIds: string[]) { const title = processTemplate( - this.args.runName ?? 'Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}' + 'runName' in this.args && this.args.runName + ? (this.args.runName as string) + : 'Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}' ) + console.log(chalk.blue(`Creating a new test run for project: ${projectCode}`)) + try { const response = await this.api.runs.createRun(projectCode, { title, description: 'Test run created through automation pipeline', type: 'static_struct', - queryPlans: [ - { - tcaseIds: tcases.data.map((t: TCaseBySeq) => t.id), - }, - ], + queryPlans: [{ tcaseIds }], }) console.log(chalk.green(`Created new test run "${title}" with ID: ${response.id}`)) + console.log( + chalk.blue(`Test run URL: ${this.baseUrl}/project/${projectCode}/run/${response.id}`) + ) return response.id } catch (error) { // Check if the error is about conflicting run ID diff --git a/src/utils/result-upload/ResultUploader.ts b/src/utils/result-upload/ResultUploader.ts index dfcb16c..bb96596 100644 --- a/src/utils/result-upload/ResultUploader.ts +++ b/src/utils/result-upload/ResultUploader.ts @@ -1,7 +1,7 @@ import { Arguments } from 'yargs' import chalk from 'chalk' import { RunTCase } from '../../api/schemas' -import { parseRunUrl, printError, printErrorThenExit, twirlLoader } from '../misc' +import { getTCaseMarker, parseRunUrl, printError, printErrorThenExit, twirlLoader } from '../misc' import { Api, createApi } from '../../api' import { Attachment, TestCaseResult } from './types' import { ResultUploadCommandArgs, UploadCommandType } from './ResultUploadCommandHandler' @@ -188,7 +188,7 @@ ${chalk.yellow('To fix this issue, choose one of the following options:')} const uploadedAttachments = await this.processConcurrently( allAttachments, async ({ attachment, tcaseIndex }) => { - const { url } = await this.api.file.uploadFile( + const { url } = await this.api.files.uploadFile( new Blob([attachment.buffer! as BlobPart]), attachment.filename ) @@ -291,8 +291,8 @@ ${chalk.yellow('To fix this issue, choose one of the following options:')} testcaseResults.forEach((result) => { if (result.name) { const tcase = testcases.find((tcase) => { - const tcaseCode = `${this.project}-${tcase.seq.toString().padStart(3, '0')}` - return result.name.includes(tcaseCode) + const tcaseMarker = getTCaseMarker(this.project, tcase.seq) + return result.name.includes(tcaseMarker) }) if (tcase) { diff --git a/src/utils/result-upload/playwrightJsonParser.ts b/src/utils/result-upload/playwrightJsonParser.ts index 6ba15ff..55ec5a7 100644 --- a/src/utils/result-upload/playwrightJsonParser.ts +++ b/src/utils/result-upload/playwrightJsonParser.ts @@ -4,7 +4,7 @@ import stripAnsi from 'strip-ansi' import { Attachment, TestCaseResult } from './types' import { Parser, ParserOptions } from './ResultUploadCommandHandler' import { ResultStatus } from '../../api/schemas' -import { parseTCaseUrl } from '../misc' +import { getTCaseMarker, parseTCaseUrl } from '../misc' import { getAttachments } from './utils' // Schema definition as per https://github.com/microsoft/playwright/blob/main/packages/playwright/types/testReporter.d.ts @@ -160,7 +160,7 @@ const getTCaseMarkerFromAnnotations = (annotations: Annotation[]) => { if (annotation.type.toLowerCase().includes('test case') && annotation.description) { const res = parseTCaseUrl(annotation.description) if (res) { - return `${res.project}-${res.tcaseSeq.toString().padStart(3, '0')}` + return getTCaseMarker(res.project, res.tcaseSeq) } } }