diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 7236a64..ea33fe1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,15 +29,15 @@ Node.js compatibility tests: `cd mnode-test && ./docker-test.sh` (requires Docke ### Entry Point & CLI Framework - `src/bin/qasphere.ts` — Entry point (`#!/usr/bin/env node`). Validates Node version, delegates to `run()`. -- `src/commands/main.ts` — Yargs setup. Registers two commands (`junit-upload`, `playwright-json-upload`) as instances of the same `ResultUploadCommandModule` class. -- `src/commands/resultUpload.ts` — `ResultUploadCommandModule` defines CLI options shared by both commands. Loads env vars, then delegates to `ResultUploadCommandHandler`. +- `src/commands/main.ts` — Yargs setup. Registers three commands (`junit-upload`, `playwright-json-upload`, `allure-upload`) as instances of the same `ResultUploadCommandModule` class. +- `src/commands/resultUpload.ts` — `ResultUploadCommandModule` defines CLI options shared by all three commands. Loads env vars, then delegates to `ResultUploadCommandHandler`. ### Core Upload Pipeline (src/utils/result-upload/) The upload flow has two stages handled by two classes: 1. **`ResultUploadCommandHandler`** — Orchestrates the overall flow: - - Parses report files using the appropriate parser (JUnit XML or Playwright JSON) + - Parses report files using the appropriate parser (JUnit XML, Playwright JSON, or Allure) - Detects project code from test case names (or from `--run-url`) - Creates a new test run (or reuses an existing one if title conflicts) - Delegates actual result uploading to `ResultUploader` @@ -51,7 +51,8 @@ The upload flow has two stages handled by two classes: - `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. - `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. -- `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by both parsers. +- `allureParser.ts` — Parses Allure results directory (`*-result.json` files). Directory-based input (not single file). Extracts test case markers from TMS links or test names. Maps `broken` → `blocked`, derives folders from suite/feature labels. +- `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by all parsers. ### API Layer (src/api/) @@ -72,12 +73,13 @@ Composable fetch wrappers using higher-order functions: Tests use **Vitest** with **MSW** (Mock Service Worker) for API mocking. Test files are in `src/tests/`: -- `result-upload.spec.ts` — Integration tests for the full upload flow (both JUnit and Playwright), with MSW intercepting all API calls +- `result-upload.spec.ts` — Integration tests for the full upload flow (JUnit, Playwright, and Allure), with MSW intercepting all API calls - `junit-xml-parsing.spec.ts` — Unit tests for JUnit XML parser - `playwright-json-parsing.spec.ts` — Unit tests for Playwright JSON parser +- `allure-parsing.spec.ts` — Unit tests for Allure results directory parser - `template-string-processing.spec.ts` — Unit tests for run name template processing -Test fixtures live in `src/tests/fixtures/` (XML files, JSON files, and mock test case data). +Test fixtures live in `src/tests/fixtures/` (XML files, JSON files, Allure result directories, and mock test case data). The `tsconfig.json` excludes `src/tests` from compilation output. diff --git a/package-lock.json b/package-lock.json index 7674e76..b98862f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "qas-cli", - "version": "0.4.4", + "version": "0.4.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "qas-cli", - "version": "0.4.4", + "version": "0.4.6", "license": "ISC", "dependencies": { "chalk": "^5.4.1", diff --git a/package.json b/package.json index e2bcf31..bf8e2c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "qas-cli", - "version": "0.4.5", + "version": "0.4.6", "description": "QAS CLI is a command line tool for submitting your automation test results to QA Sphere at https://qasphere.com/", "type": "module", "main": "./build/bin/qasphere.js", diff --git a/src/commands/main.ts b/src/commands/main.ts index 98ed8a3..05af261 100644 --- a/src/commands/main.ts +++ b/src/commands/main.ts @@ -13,6 +13,7 @@ Required variables: ${qasEnvs.join(', ')} ) .command(new ResultUploadCommandModule('junit-upload')) .command(new ResultUploadCommandModule('playwright-json-upload')) + .command(new ResultUploadCommandModule('allure-upload')) .demandCommand(1, '') .help('h') .alias('h', 'help') diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index 78b6687..40fa7fd 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -10,11 +10,13 @@ import { const commandTypeDisplayStrings: Record = { 'junit-upload': 'JUnit XML', 'playwright-json-upload': 'Playwright JSON', + 'allure-upload': 'Allure', } const commandTypeFileExtensions: Record = { 'junit-upload': 'xml', 'playwright-json-upload': 'json', + 'allure-upload': '', } export class ResultUploadCommandModule implements CommandModule { @@ -25,7 +27,8 @@ export class ResultUploadCommandModule implements CommandModule { @@ -82,40 +85,33 @@ export class ResultUploadCommandModule implements CommandModule { + test('Should parse results directory with passed/failed/broken/skipped tests', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + // Should only parse *-result.json files, not containers or attachments + expect(testcases.length).toBeGreaterThanOrEqual(12) + + const statuses = testcases.map((tc) => tc.status) + expect(statuses).toContain('passed') + expect(statuses).toContain('failed') + expect(statuses).toContain('blocked') // broken maps to blocked + expect(statuses).toContain('skipped') + }) + + test('Should map broken status to blocked', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const brokenTest = testcases.find((tc) => tc.name === 'Database connection test') + expect(brokenTest).toBeDefined() + expect(brokenTest!.status).toBe('blocked') + }) + + test('Should map unknown status to passed', async () => { + const tmpDir = join('/tmp', 'allure-unknown-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'aaa-result.json'), + JSON.stringify({ + name: 'Unknown status test', + status: 'unknown', + uuid: 'unknown-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].status).toBe('passed') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should derive folder from suite label (highest priority)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const suiteTest = testcases.find((tc) => tc.name === 'Login with valid credentials') + expect(suiteTest).toBeDefined() + // Has both parentSuite="tests" and suite="login_test" — suite should win + expect(suiteTest!.folder).toBe('login_test') + }) + + test('Should derive folder from parentSuite when suite is absent', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const parentSuiteTest = testcases.find((tc) => tc.name === 'Mocha style test') + expect(parentSuiteTest).toBeDefined() + expect(parentSuiteTest!.folder).toBe('API Tests') + }) + + test('Should derive folder from feature label (behave style)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const featureTest = testcases.find((tc) => tc.name === 'User can register with valid email') + expect(featureTest).toBeDefined() + expect(featureTest!.folder).toBe('Registration') + }) + + test('Should derive empty folder when no labels present (jest style)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const noLabelsTest = testcases.find((tc) => tc.name === 'Jest style test with no labels') + expect(noLabelsTest).toBeDefined() + expect(noLabelsTest!.folder).toBe('') + }) + + test('Should calculate duration from start/stop timestamps', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const loginTest = testcases.find((tc) => tc.name === 'Login with valid credentials') + expect(loginTest).toBeDefined() + expect(loginTest!.timeTaken).toBe(1500) // 1700000001500 - 1700000000000 + }) + + test('Should extract error message from statusDetails.message', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const failedTest = testcases.find((tc) => tc.name.includes('Login with invalid password')) + expect(failedTest).toBeDefined() + expect(failedTest!.message).toContain('AssertionError') + expect(failedTest!.message).toContain('Message:') + }) + + test('Should extract trace from statusDetails.trace', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const failedTest = testcases.find((tc) => tc.name.includes('Login with invalid password')) + expect(failedTest).toBeDefined() + expect(failedTest!.message).toContain('Trace:') + expect(failedTest!.message).toContain('Traceback') + }) + + test('Should handle null statusDetails gracefully', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const passedTest = testcases.find((tc) => tc.name === 'Login with valid credentials') + expect(passedTest).toBeDefined() + expect(passedTest!.message).toBe('') + }) + + test('Should handle empty statusDetails object gracefully', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const emptyDetailsTest = testcases.find((tc) => tc.name === 'Test with empty statusDetails') + expect(emptyDetailsTest).toBeDefined() + expect(emptyDetailsTest!.message).toBe('') + }) + + test('Should read attachment files via getAttachments', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const testWithAttachment = testcases.find((tc) => + tc.name.includes('Login with invalid password') + ) + expect(testWithAttachment).toBeDefined() + expect(testWithAttachment!.attachments).toHaveLength(1) + expect(testWithAttachment!.attachments[0].filename).toBe('ut-screenshot-attachment.txt') + expect(testWithAttachment!.attachments[0].buffer).not.toBeNull() + }) + + test('Should handle results with no attachments', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const noAttachTest = testcases.find((tc) => tc.name === 'Database connection test') + expect(noAttachTest).toBeDefined() + expect(noAttachTest!.attachments).toHaveLength(0) + }) + + test('Should extract test case marker from test name', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + // TEST-002 in the name is extracted by the upstream pipeline, not by the parser itself + // The parser just passes the name through + const markerTest = testcases.find((tc) => tc.name.includes('Login with invalid password')) + expect(markerTest).toBeDefined() + expect(markerTest!.name).toContain('TEST-002') + }) + + test('Should extract test case marker from TMS link URL (QA Sphere)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const tmsUrlTest = testcases.find((tc) => tc.name.includes('Login test via TMS URL')) + expect(tmsUrlTest).toBeDefined() + // TMS link URL is a QA Sphere URL — marker should be prefixed + expect(tmsUrlTest!.name).toBe('PRJ-123: Login test via TMS URL') + }) + + test('Should extract test case marker from TMS link name (non-QA Sphere URL)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const tmsNameTest = testcases.find((tc) => tc.name.includes('Checkout test via TMS name')) + expect(tmsNameTest).toBeDefined() + // URL isn't a QA Sphere URL, so falls back to regex on name + expect(tmsNameTest!.name).toBe('TESTCASE-456: Checkout test via TMS name') + }) + + test('Should handle parameterized tests as separate results', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + const paramTests = testcases.filter((tc) => tc.name.startsWith('Test login[')) + expect(paramTests).toHaveLength(2) + + const chromiumTest = paramTests.find((tc) => tc.name.includes('chromium')) + const firefoxTest = paramTests.find((tc) => tc.name.includes('firefox')) + expect(chromiumTest!.status).toBe('passed') + expect(firefoxTest!.status).toBe('failed') + }) + + test('Should skip container files (only parse *-result.json)', async () => { + const testcases = await parseAllureResults(allureBasePath, allureBasePath, defaultOptions) + + // Container files should not produce any test results + // All results should have meaningful names (no container-like names) + for (const tc of testcases) { + expect(tc.name).toBeDefined() + expect(tc.name.length).toBeGreaterThan(0) + } + }) + + test('Should handle empty results directory', async () => { + const tmpDir = join('/tmp', 'allure-empty-test') + mkdirSync(tmpDir, { recursive: true }) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(0) + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should ignore non-result files in directory', async () => { + const tmpDir = join('/tmp', 'allure-nonresult-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync(join(tmpDir, 'something-testsuite.xml'), 'legacy') + writeFileSync(join(tmpDir, 'image-attachment.png'), 'fake png data') + writeFileSync( + join(tmpDir, 'valid-result.json'), + JSON.stringify({ + name: 'Valid test', + status: 'passed', + uuid: 'valid-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('Valid test') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should skip malformed JSON with warning', async () => { + const tmpDir = join('/tmp', 'allure-malformed-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync(join(tmpDir, 'bad-result.json'), '{ not valid json }}}') + writeFileSync( + join(tmpDir, 'good-result.json'), + JSON.stringify({ + name: 'Good test', + status: 'passed', + uuid: 'good-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('Good test') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should skip files that fail Zod validation with warning', async () => { + const tmpDir = join('/tmp', 'allure-invalid-schema-test') + mkdirSync(tmpDir, { recursive: true }) + // Missing required 'status' field + writeFileSync( + join(tmpDir, 'invalid-result.json'), + JSON.stringify({ + name: 'Invalid test', + uuid: 'invalid-uuid', + start: 1700000000000, + stop: 1700000001000, + }) + ) + writeFileSync( + join(tmpDir, 'valid-result.json'), + JSON.stringify({ + name: 'Valid test', + status: 'passed', + uuid: 'valid-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('Valid test') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should skip message for passed tests when skipStdout is on-success', async () => { + const tmpDir = join('/tmp', 'allure-skip-stdout-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'passed-result.json'), + JSON.stringify({ + name: 'Passed test with details', + status: 'passed', + uuid: 'skip-stdout-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + statusDetails: { + message: 'Some info message', + }, + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, { + skipStdout: 'on-success', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].message).not.toContain('Some info message') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should skip trace for passed tests when skipStderr is on-success', async () => { + const tmpDir = join('/tmp', 'allure-skip-stderr-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'passed-result.json'), + JSON.stringify({ + name: 'Passed test with trace', + status: 'passed', + uuid: 'skip-stderr-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + statusDetails: { + message: 'Some info message', + trace: 'Some trace info', + }, + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, { + skipStdout: 'never', + skipStderr: 'on-success', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].message).toContain('Some info message') + expect(testcases[0].message).not.toContain('Some trace info') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should include message/trace for failed tests even when skip options are on-success', async () => { + const tmpDir = join('/tmp', 'allure-failed-skip-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'failed-result.json'), + JSON.stringify({ + name: 'Failed test with details', + status: 'failed', + uuid: 'failed-skip-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + statusDetails: { + message: 'Failure message', + trace: 'Failure trace', + }, + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, { + skipStdout: 'on-success', + skipStderr: 'on-success', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].message).toContain('Failure message') + expect(testcases[0].message).toContain('Failure trace') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should handle results with null arrays gracefully', async () => { + const tmpDir = join('/tmp', 'allure-null-arrays-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'null-arrays-result.json'), + JSON.stringify({ + name: 'Test with null arrays', + status: 'passed', + uuid: 'null-arrays-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: null, + attachments: null, + links: null, + parameters: null, + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('Test with null arrays') + expect(testcases[0].folder).toBe('') + expect(testcases[0].attachments).toHaveLength(0) + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) + + test('Should escape HTML in message/trace', async () => { + const tmpDir = join('/tmp', 'allure-html-escape-test') + mkdirSync(tmpDir, { recursive: true }) + writeFileSync( + join(tmpDir, 'html-result.json'), + JSON.stringify({ + name: 'Test with HTML chars', + status: 'failed', + uuid: 'html-uuid', + start: 1700000000000, + stop: 1700000001000, + labels: [], + statusDetails: { + message: 'Expected
to contain "hello"', + trace: 'at :1:1', + }, + }) + ) + + try { + const testcases = await parseAllureResults(tmpDir, tmpDir, defaultOptions) + expect(testcases).toHaveLength(1) + expect(testcases[0].message).toContain('<div>') + expect(testcases[0].message).toContain('<anonymous>') + expect(testcases[0].message).not.toContain('
') + } finally { + rmSync(tmpDir, { recursive: true }) + } + }) +}) diff --git a/src/tests/fixtures/allure/empty-tsuite/aaa-result.json b/src/tests/fixtures/allure/empty-tsuite/aaa-result.json new file mode 100644 index 0000000..59a7553 --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/aaa-result.json @@ -0,0 +1,8 @@ +{ + "name": "Test cart TEST-002", + "status": "passed", + "uuid": "empty-uuid-1", + "start": 1700000000000, + "stop": 1700000001000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/fixtures/allure/empty-tsuite/bbb-container.json b/src/tests/fixtures/allure/empty-tsuite/bbb-container.json new file mode 100644 index 0000000..4a08695 --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/bbb-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "container-uuid-1", + "children": ["empty-uuid-1"], + "befores": [{ "name": "setup_fixture", "status": "passed" }], + "afters": [], + "start": 1700000000000, + "stop": 1700000001000 +} diff --git a/src/tests/fixtures/allure/matching-tcases/aaa-attachment.json b/src/tests/fixtures/allure/matching-tcases/aaa-attachment.json new file mode 100644 index 0000000..7d2950c --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/aaa-attachment.json @@ -0,0 +1 @@ +{ "test": "data" } diff --git a/src/tests/fixtures/allure/matching-tcases/aaa-result.json b/src/tests/fixtures/allure/matching-tcases/aaa-result.json new file mode 100644 index 0000000..a70f2bc --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/aaa-result.json @@ -0,0 +1,11 @@ +{ + "name": "Test cart TEST-002", + "status": "passed", + "uuid": "aaa-uuid-1", + "start": 1700000000000, + "stop": 1700000001000, + "labels": [{ "name": "suite", "value": "ui.cart" }], + "attachments": [ + { "name": "result data", "source": "aaa-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/bbb-result.json b/src/tests/fixtures/allure/matching-tcases/bbb-result.json new file mode 100644 index 0000000..09207c9 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/bbb-result.json @@ -0,0 +1,11 @@ +{ + "name": "Test checkout TEST-003", + "status": "passed", + "uuid": "bbb-uuid-2", + "start": 1700000001000, + "stop": 1700000002000, + "labels": [{ "name": "suite", "value": "ui.cart" }], + "attachments": [ + { "name": "result data", "source": "aaa-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/ccc-result.json b/src/tests/fixtures/allure/matching-tcases/ccc-result.json new file mode 100644 index 0000000..daf6645 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/ccc-result.json @@ -0,0 +1,11 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "ccc-uuid-3", + "start": 1700000002000, + "stop": 1700000003000, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "result data", "source": "aaa-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/ddd-result.json b/src/tests/fixtures/allure/matching-tcases/ddd-result.json new file mode 100644 index 0000000..ebbe945 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/ddd-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "ddd-uuid-4", + "start": 1700000003000, + "stop": 1700000004000, + "statusDetails": { + "message": "AssertionError: expected true to equal false" + }, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "result data", "source": "aaa-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/eee-result.json b/src/tests/fixtures/allure/matching-tcases/eee-result.json new file mode 100644 index 0000000..7dec6c7 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/eee-result.json @@ -0,0 +1,11 @@ +{ + "name": "Menu page content TEST-007", + "status": "passed", + "uuid": "eee-uuid-5", + "start": 1700000004000, + "stop": 1700000005000, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "result data", "source": "aaa-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/aaa-result.json b/src/tests/fixtures/allure/missing-attachments/aaa-result.json new file mode 100644 index 0000000..8118db6 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/aaa-result.json @@ -0,0 +1,11 @@ +{ + "name": "Test cart TEST-002", + "status": "passed", + "uuid": "ma-uuid-1", + "start": 1700000000000, + "stop": 1700000001000, + "labels": [{ "name": "suite", "value": "ui.cart" }], + "attachments": [ + { "name": "result data", "source": "existing-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/bbb-result.json b/src/tests/fixtures/allure/missing-attachments/bbb-result.json new file mode 100644 index 0000000..f9c2fd7 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/bbb-result.json @@ -0,0 +1,11 @@ +{ + "name": "Test checkout TEST-003", + "status": "passed", + "uuid": "ma-uuid-2", + "start": 1700000001000, + "stop": 1700000002000, + "labels": [{ "name": "suite", "value": "ui.cart" }], + "attachments": [ + { "name": "result data", "source": "existing-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/ccc-result.json b/src/tests/fixtures/allure/missing-attachments/ccc-result.json new file mode 100644 index 0000000..fc8be38 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/ccc-result.json @@ -0,0 +1,11 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "ma-uuid-3", + "start": 1700000002000, + "stop": 1700000003000, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "screenshot", "source": "nonexistent-attachment.png", "type": "image/png" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/ddd-result.json b/src/tests/fixtures/allure/missing-attachments/ddd-result.json new file mode 100644 index 0000000..12ea35a --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/ddd-result.json @@ -0,0 +1,12 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "ma-uuid-4", + "start": 1700000003000, + "stop": 1700000004000, + "statusDetails": { "message": "AssertionError" }, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "result data", "source": "existing-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/eee-result.json b/src/tests/fixtures/allure/missing-attachments/eee-result.json new file mode 100644 index 0000000..a118992 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/eee-result.json @@ -0,0 +1,11 @@ +{ + "name": "Menu page content TEST-007", + "status": "passed", + "uuid": "ma-uuid-5", + "start": 1700000004000, + "stop": 1700000005000, + "labels": [{ "name": "suite", "value": "ui.contents" }], + "attachments": [ + { "name": "result data", "source": "existing-attachment.json", "type": "application/json" } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/existing-attachment.json b/src/tests/fixtures/allure/missing-attachments/existing-attachment.json new file mode 100644 index 0000000..7d2950c --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/existing-attachment.json @@ -0,0 +1 @@ +{ "test": "data" } diff --git a/src/tests/fixtures/allure/missing-tcases/aaa-result.json b/src/tests/fixtures/allure/missing-tcases/aaa-result.json new file mode 100644 index 0000000..43496e0 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/aaa-result.json @@ -0,0 +1,8 @@ +{ + "name": "Test cart TEST-002", + "status": "passed", + "uuid": "miss-uuid-1", + "start": 1700000000000, + "stop": 1700000001000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/fixtures/allure/missing-tcases/bbb-result.json b/src/tests/fixtures/allure/missing-tcases/bbb-result.json new file mode 100644 index 0000000..7d973e4 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/bbb-result.json @@ -0,0 +1,8 @@ +{ + "name": "Test checkout TEST-003", + "status": "passed", + "uuid": "miss-uuid-2", + "start": 1700000001000, + "stop": 1700000002000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/fixtures/allure/missing-tcases/ccc-result.json b/src/tests/fixtures/allure/missing-tcases/ccc-result.json new file mode 100644 index 0000000..782abde --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/ccc-result.json @@ -0,0 +1,8 @@ +{ + "name": "TEST-000 Missing content", + "status": "passed", + "uuid": "miss-uuid-3", + "start": 1700000002000, + "stop": 1700000003000, + "labels": [{ "name": "suite", "value": "ui.contents" }] +} diff --git a/src/tests/fixtures/allure/missing-tcases/eee-result.json b/src/tests/fixtures/allure/missing-tcases/eee-result.json new file mode 100644 index 0000000..2409326 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/eee-result.json @@ -0,0 +1,8 @@ +{ + "name": "About page content", + "status": "passed", + "uuid": "miss-uuid-5", + "start": 1700000004000, + "stop": 1700000005000, + "labels": [{ "name": "suite", "value": "ui.contents" }] +} diff --git a/src/tests/fixtures/allure/missing-tcases/fff-result.json b/src/tests/fixtures/allure/missing-tcases/fff-result.json new file mode 100644 index 0000000..9c78270 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/fff-result.json @@ -0,0 +1,9 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "miss-uuid-6", + "start": 1700000005000, + "stop": 1700000006000, + "statusDetails": { "message": "AssertionError" }, + "labels": [{ "name": "suite", "value": "ui.contents" }] +} diff --git a/src/tests/fixtures/allure/missing-tcases/ggg-result.json b/src/tests/fixtures/allure/missing-tcases/ggg-result.json new file mode 100644 index 0000000..6426563 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/ggg-result.json @@ -0,0 +1,8 @@ +{ + "name": "Menu page content TEST-007", + "status": "passed", + "uuid": "miss-uuid-7", + "start": 1700000006000, + "stop": 1700000007000, + "labels": [{ "name": "suite", "value": "ui.contents" }] +} diff --git a/src/tests/fixtures/allure/unit-tests/broken-result.json b/src/tests/fixtures/allure/unit-tests/broken-result.json new file mode 100644 index 0000000..8ff96de --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/broken-result.json @@ -0,0 +1,11 @@ +{ + "name": "Database connection test", + "status": "broken", + "uuid": "ut-uuid-3", + "start": 1700000004000, + "stop": 1700000004500, + "labels": [{ "name": "suite", "value": "db_tests" }], + "statusDetails": { + "message": "ConnectionRefusedError: [Errno 111] Connection refused" + } +} diff --git a/src/tests/fixtures/allure/unit-tests/empty-statusdetails-result.json b/src/tests/fixtures/allure/unit-tests/empty-statusdetails-result.json new file mode 100644 index 0000000..3844b06 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/empty-statusdetails-result.json @@ -0,0 +1,9 @@ +{ + "name": "Test with empty statusDetails", + "status": "failed", + "uuid": "ut-uuid-10", + "start": 1700000016000, + "stop": 1700000017000, + "labels": [{ "name": "suite", "value": "misc" }], + "statusDetails": {} +} diff --git a/src/tests/fixtures/allure/unit-tests/failed-message-trace-result.json b/src/tests/fixtures/allure/unit-tests/failed-message-trace-result.json new file mode 100644 index 0000000..1f8c85c --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/failed-message-trace-result.json @@ -0,0 +1,20 @@ +{ + "name": "TEST-002 Login with invalid password", + "status": "failed", + "uuid": "ut-uuid-2", + "start": 1700000002000, + "stop": 1700000003000, + "fullName": "tests.login_test.TestLogin#test_invalid_password", + "labels": [{ "name": "suite", "value": "login_test" }], + "statusDetails": { + "message": "AssertionError: expected 'Welcome' to equal 'Error'", + "trace": "Traceback (most recent call last):\n File \"test_login.py\", line 42\n assert title == 'Error'\nAssertionError" + }, + "attachments": [ + { + "name": "Screenshot on failure", + "source": "ut-screenshot-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/unit-tests/feature-label-result.json b/src/tests/fixtures/allure/unit-tests/feature-label-result.json new file mode 100644 index 0000000..fe87e80 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/feature-label-result.json @@ -0,0 +1,11 @@ +{ + "name": "User can register with valid email", + "status": "passed", + "uuid": "ut-uuid-5", + "start": 1700000006000, + "stop": 1700000007000, + "labels": [ + { "name": "feature", "value": "Registration" }, + { "name": "severity", "value": "critical" } + ] +} diff --git a/src/tests/fixtures/allure/unit-tests/no-labels-result.json b/src/tests/fixtures/allure/unit-tests/no-labels-result.json new file mode 100644 index 0000000..c29a70c --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/no-labels-result.json @@ -0,0 +1,8 @@ +{ + "name": "Jest style test with no labels", + "status": "passed", + "uuid": "ut-uuid-7", + "start": 1700000010000, + "stop": 1700000011000, + "labels": [] +} diff --git a/src/tests/fixtures/allure/unit-tests/parameterized-a-result.json b/src/tests/fixtures/allure/unit-tests/parameterized-a-result.json new file mode 100644 index 0000000..c51fb38 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/parameterized-a-result.json @@ -0,0 +1,10 @@ +{ + "name": "Test login[chromium]", + "status": "passed", + "uuid": "ut-uuid-11a", + "start": 1700000018000, + "stop": 1700000019000, + "fullName": "tests.login_test#test_login", + "labels": [{ "name": "suite", "value": "login_test" }], + "parameters": [{ "name": "browser_name", "value": "chromium" }] +} diff --git a/src/tests/fixtures/allure/unit-tests/parameterized-b-result.json b/src/tests/fixtures/allure/unit-tests/parameterized-b-result.json new file mode 100644 index 0000000..3f17b02 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/parameterized-b-result.json @@ -0,0 +1,13 @@ +{ + "name": "Test login[firefox]", + "status": "failed", + "uuid": "ut-uuid-11b", + "start": 1700000020000, + "stop": 1700000021000, + "fullName": "tests.login_test#test_login", + "labels": [{ "name": "suite", "value": "login_test" }], + "parameters": [{ "name": "browser_name", "value": "firefox" }], + "statusDetails": { + "message": "Element not found" + } +} diff --git a/src/tests/fixtures/allure/unit-tests/parentSuite-label-result.json b/src/tests/fixtures/allure/unit-tests/parentSuite-label-result.json new file mode 100644 index 0000000..aaa0f8d --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/parentSuite-label-result.json @@ -0,0 +1,11 @@ +{ + "name": "Mocha style test", + "status": "passed", + "uuid": "ut-uuid-6", + "start": 1700000008000, + "stop": 1700000009000, + "labels": [ + { "name": "parentSuite", "value": "API Tests" }, + { "name": "package", "value": "api.tests" } + ] +} diff --git a/src/tests/fixtures/allure/unit-tests/passed-suite-result.json b/src/tests/fixtures/allure/unit-tests/passed-suite-result.json new file mode 100644 index 0000000..c6899dc --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/passed-suite-result.json @@ -0,0 +1,18 @@ +{ + "name": "Login with valid credentials", + "status": "passed", + "uuid": "ut-uuid-1", + "start": 1700000000000, + "stop": 1700000001500, + "fullName": "tests.login_test.TestLogin#test_valid_login", + "labels": [ + { "name": "parentSuite", "value": "tests" }, + { "name": "suite", "value": "login_test" }, + { "name": "subSuite", "value": "TestLogin" }, + { "name": "framework", "value": "pytest" } + ], + "statusDetails": null, + "attachments": [], + "links": [], + "parameters": [] +} diff --git a/src/tests/fixtures/allure/unit-tests/skipped-result.json b/src/tests/fixtures/allure/unit-tests/skipped-result.json new file mode 100644 index 0000000..02f3214 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/skipped-result.json @@ -0,0 +1,11 @@ +{ + "name": "Feature flag disabled test", + "status": "skipped", + "uuid": "ut-uuid-4", + "start": 1700000005000, + "stop": 1700000005000, + "labels": [{ "name": "suite", "value": "feature_tests" }], + "statusDetails": { + "message": "Skipped: feature flag is disabled" + } +} diff --git a/src/tests/fixtures/allure/unit-tests/tms-link-name-result.json b/src/tests/fixtures/allure/unit-tests/tms-link-name-result.json new file mode 100644 index 0000000..6412deb --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/tms-link-name-result.json @@ -0,0 +1,11 @@ +{ + "name": "Checkout test via TMS name", + "status": "passed", + "uuid": "ut-uuid-9", + "start": 1700000014000, + "stop": 1700000015000, + "labels": [{ "name": "suite", "value": "checkout" }], + "links": [ + { "type": "tms", "url": "https://jira.example.com/browse/TESTCASE-456", "name": "TESTCASE-456" } + ] +} diff --git a/src/tests/fixtures/allure/unit-tests/tms-link-url-result.json b/src/tests/fixtures/allure/unit-tests/tms-link-url-result.json new file mode 100644 index 0000000..cf790e5 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/tms-link-url-result.json @@ -0,0 +1,15 @@ +{ + "name": "Login test via TMS URL", + "status": "passed", + "uuid": "ut-uuid-8", + "start": 1700000012000, + "stop": 1700000013000, + "labels": [{ "name": "suite", "value": "login" }], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/PRJ/tcase/123", + "name": "PRJ-123" + } + ] +} diff --git a/src/tests/fixtures/allure/unit-tests/ut-screenshot-attachment.txt b/src/tests/fixtures/allure/unit-tests/ut-screenshot-attachment.txt new file mode 100644 index 0000000..afed121 --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/ut-screenshot-attachment.txt @@ -0,0 +1 @@ +fake screenshot data \ No newline at end of file diff --git a/src/tests/fixtures/allure/unit-tests/zzz-container.json b/src/tests/fixtures/allure/unit-tests/zzz-container.json new file mode 100644 index 0000000..04d8e5e --- /dev/null +++ b/src/tests/fixtures/allure/unit-tests/zzz-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "container-uuid-999", + "children": ["ut-uuid-1"], + "befores": [{ "name": "setup", "status": "passed" }], + "afters": [], + "start": 1700000000000, + "stop": 1700000001000 +} diff --git a/src/tests/fixtures/allure/without-markers/aaa-result.json b/src/tests/fixtures/allure/without-markers/aaa-result.json new file mode 100644 index 0000000..3572aa9 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/aaa-result.json @@ -0,0 +1,8 @@ +{ + "name": "Test cart TEST-002", + "status": "passed", + "uuid": "wm-uuid-1", + "start": 1700000000000, + "stop": 1700000001000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/fixtures/allure/without-markers/bbb-result.json b/src/tests/fixtures/allure/without-markers/bbb-result.json new file mode 100644 index 0000000..2c441e8 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/bbb-result.json @@ -0,0 +1,8 @@ +{ + "name": "The cart is still filled after refreshing the page", + "status": "passed", + "uuid": "wm-uuid-2", + "start": 1700000001000, + "stop": 1700000002000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/fixtures/allure/without-markers/ccc-result.json b/src/tests/fixtures/allure/without-markers/ccc-result.json new file mode 100644 index 0000000..b8bc3b7 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/ccc-result.json @@ -0,0 +1,8 @@ +{ + "name": "TEST-010: Cart should be cleared after making the checkout", + "status": "passed", + "uuid": "wm-uuid-3", + "start": 1700000002000, + "stop": 1700000003000, + "labels": [{ "name": "suite", "value": "ui.cart" }] +} diff --git a/src/tests/result-upload.spec.ts b/src/tests/result-upload.spec.ts index 3aee549..2aa4e76 100644 --- a/src/tests/result-upload.spec.ts +++ b/src/tests/result-upload.spec.ts @@ -159,29 +159,59 @@ const cleanupGeneratedMappingFiles = (existingMappingFiles?: Set) => { }) } +const expectedCounts = { + matchingResults: 5, + matchingAttachments: 5, + missingTcasesMatchedResults: 4, + missingAttExistingUploads: 4, + missingAttTotalResults: 5, + withoutMarkersTotal: 3, + withoutMarkersNewTcases: 2, +} + const fileTypes = [ { name: 'JUnit XML', command: 'junit-upload', dataBasePath: './src/tests/fixtures/junit-xml', fileExtension: 'xml', + inputType: 'file' as const, + expected: { ...expectedCounts }, }, { name: 'Playwright JSON', command: 'playwright-json-upload', dataBasePath: './src/tests/fixtures/playwright-json', fileExtension: 'json', + inputType: 'file' as const, + expected: { ...expectedCounts }, + }, + { + name: 'Allure', + command: 'allure-upload', + dataBasePath: './src/tests/fixtures/allure', + fileExtension: '', + inputType: 'directory' as const, + expected: { ...expectedCounts }, }, ] +const getFixturePath = (fileType: (typeof fileTypes)[number], fixtureName: string) => + fileType.inputType === 'directory' + ? `${fileType.dataBasePath}/${fixtureName}` + : `${fileType.dataBasePath}/${fixtureName}.${fileType.fileExtension}` + fileTypes.forEach((fileType) => { describe(`Uploading ${fileType.name} files`, () => { + const e = fileType.expected + describe('Argument parsing', () => { test('Passing correct Run URL pattern should result in success', async () => { + const fixture = getFixturePath(fileType, 'matching-tcases') const patterns = [ - `${fileType.command} --run-url ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/ ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} --run-url ${runURL} ${fixture}`, + `${fileType.command} -r ${runURL}/ ${fixture}`, + `${fileType.command} -r ${runURL}/tcase/1 ${fixture}`, ] for (const pattern of patterns) { @@ -189,7 +219,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() await run(pattern) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(1) // 5 results total + expect(numResultUploadCalls()).toBe(Math.ceil(e.matchingResults / 50)) } }) @@ -198,16 +228,17 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${getFixturePath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(5) // 5 results total + expect(numResultUploadCalls()).toBe(e.matchingResults) }) test('Passing incorrect Run URL pattern should result in failure', async () => { + const fixture = getFixturePath(fileType, 'matching-tcases') const patterns = [ - `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}abc/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fixture}`, + `${fileType.command} -r ${runURL}abc/tcase/1 ${fixture}`, ] for (const pattern of patterns) { @@ -232,20 +263,16 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) - await run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` - ) + await run(`${fileType.command} -r ${runURL} ${getFixturePath(fileType, 'matching-tcases')}`) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(3) // 5 results total + expect(numResultUploadCalls()).toBe(Math.ceil(e.matchingResults / 2)) }) test('Test cases on reports with a missing test case on QAS should throw an error', async () => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await expect( - run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` - ) + run(`${fileType.command} -r ${runURL} ${getFixturePath(fileType, 'missing-tcases')}`) ).rejects.toThrowError() expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(0) @@ -256,20 +283,20 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${getFixturePath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(2) // 4 results total + expect(numResultUploadCalls()).toBe(Math.ceil(e.missingTcasesMatchedResults / 3)) }) test('Test cases on reports with missing test cases should be successful with --ignore-unmatched', async () => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --ignore-unmatched ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --ignore-unmatched ${getFixturePath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(1) // 4 results total + expect(numResultUploadCalls()).toBe(Math.ceil(e.missingTcasesMatchedResults / 50)) }) test('Test cases from multiple reports should be processed successfully', async () => { @@ -277,17 +304,17 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${getFixturePath(fileType, 'missing-tcases')} ${getFixturePath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(4) // 8 results total + expect(numResultUploadCalls()).toBe(Math.ceil((e.missingTcasesMatchedResults * 2) / 2)) }) test('Test suite with empty tcases should not result in error and be skipped', async () => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/empty-tsuite.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${getFixturePath(fileType, 'empty-tsuite')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(1) // 1 result total @@ -300,17 +327,17 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${getFixturePath(fileType, 'matching-tcases')}` ) - expect(numFileUploadCalls()).toBe(5) - expect(numResultUploadCalls()).toBe(2) // 5 results total + expect(numFileUploadCalls()).toBe(e.matchingAttachments) + expect(numResultUploadCalls()).toBe(Math.ceil(e.matchingResults / 3)) }) test('Missing attachments should throw an error', async () => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await expect( run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${getFixturePath(fileType, 'missing-attachments')}` ) ).rejects.toThrow() expect(numFileUploadCalls()).toBe(0) @@ -321,10 +348,10 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${runURL} --attachments --force ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments --force ${getFixturePath(fileType, 'missing-attachments')}` ) - expect(numFileUploadCalls()).toBe(4) - expect(numResultUploadCalls()).toBe(5) // 5 results total + expect(numFileUploadCalls()).toBe(e.missingAttExistingUploads) + expect(numResultUploadCalls()).toBe(e.missingAttTotalResults) }) }) @@ -342,7 +369,7 @@ fileTypes.forEach((fileType) => { try { // This should create a new run since no --run-url is specified await run( - `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${getFixturePath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('CI Build 456') @@ -363,7 +390,7 @@ fileTypes.forEach((fileType) => { const expectedDay = String(now.getDate()).padStart(2, '0') await run( - `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${getFixturePath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe(`Test Run ${expectedYear}-${expectedMonth}-${expectedDay}`) @@ -375,7 +402,7 @@ fileTypes.forEach((fileType) => { try { await run( - `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${getFixturePath(fileType, 'matching-tcases')}` ) const now = new Date() @@ -398,18 +425,16 @@ fileTypes.forEach((fileType) => { createRunTitleConflict = true await run( - `${fileType.command} --run-name "duplicate run title" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "duplicate run title" ${getFixturePath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('duplicate run title') expect(numFileUploadCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(1) // 5 results total + expect(numResultUploadCalls()).toBe(Math.ceil(e.matchingResults / 50)) }) test('Should use default name template when --run-name is not specified', async () => { - await run( - `${fileType.command} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` - ) + await run(`${fileType.command} ${getFixturePath(fileType, 'matching-tcases')}`) // Should use default format: "Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}" expect(lastCreatedRunTitle).toContain('Automated test run - ') @@ -447,10 +472,10 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${getFixturePath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) - expect(numResultUploadCalls()).toBe(3) // 3 results total + expect(numResultUploadCalls()).toBe(e.withoutMarkersTotal) }) test('Should not create new test case if one with same title already exists', async () => { @@ -484,10 +509,10 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${getFixturePath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) - expect(numResultUploadCalls()).toBe(3) // 3 results total + expect(numResultUploadCalls()).toBe(e.withoutMarkersTotal) }) test('Should not create new test cases if all results have valid markers', async () => { @@ -496,10 +521,10 @@ fileTypes.forEach((fileType) => { setMaxResultsInRequest(1) await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${getFixturePath(fileType, 'matching-tcases')}` ) expect(numCreateTCasesCalls()).toBe(0) - expect(numResultUploadCalls()).toBe(5) // 5 results total + expect(numResultUploadCalls()).toBe(e.matchingResults) }) }) }) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index f4b90be..f839de6 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -9,8 +9,9 @@ import { TestCaseResult } from './types' import { ResultUploader } from './ResultUploader' import { parseJUnitXml } from './junitXmlParser' import { parsePlaywrightJson } from './playwrightJsonParser' +import { parseAllureResults } from './allureParser' -export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' +export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' | 'allure-upload' export type SkipOutputOption = 'on-success' | 'never' @@ -20,6 +21,8 @@ export interface ParserOptions { } export type Parser = ( + // Primary input string: file content for file-based parsers, directory path for + // directory-based parsers (e.g., Allure) data: string, attachmentBaseDirectory: string, options: ParserOptions @@ -62,8 +65,11 @@ const DEFAULT_MAPPING_FILENAME_TEMPLATE = 'qasphere-automapping-{YYYY}{MM}{DD}-{ const commandTypeParsers: Record = { 'junit-upload': parseJUnitXml, 'playwright-json-upload': parsePlaywrightJson, + 'allure-upload': parseAllureResults, } +const directoryInputTypes: Set = new Set(['allure-upload']) + export class ResultUploadCommandHandler { private api: Api private baseUrl: string @@ -132,10 +138,21 @@ export class ResultUploadCommandHandler { } for (const file of this.args.files) { - const fileData = readFileSync(file).toString() + let fileData: string + let attachmentBaseDir: string + + if (directoryInputTypes.has(this.type)) { + // For directory-based parsers (e.g., Allure), pass the directory path directly + fileData = file + attachmentBaseDir = file + } else { + fileData = readFileSync(file).toString() + attachmentBaseDir = dirname(file) + } + const fileResults = await commandTypeParsers[this.type]( fileData, - dirname(file), + attachmentBaseDir, parserOptions ) results.push({ file, results: fileResults }) diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts new file mode 100644 index 0000000..0d91dd8 --- /dev/null +++ b/src/utils/result-upload/allureParser.ts @@ -0,0 +1,244 @@ +import z from 'zod' +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import escapeHtml from 'escape-html' +import { TestCaseResult } from './types' +import { Parser, ParserOptions } from './ResultUploadCommandHandler' +import { ResultStatus } from '../../api/schemas' +import { getTCaseMarker, parseTCaseUrl } from '../misc' +import { getAttachments } from './utils' + +// Zod schemas for Allure result files + +const allureStatusSchema = z.enum(['passed', 'failed', 'broken', 'skipped', 'unknown']) + +const allureStatusDetailsSchema = z + .object({ + message: z.string().optional(), + trace: z.string().optional(), + known: z.boolean().optional(), + muted: z.boolean().optional(), + flaky: z.boolean().optional(), + }) + .nullable() + .optional() + +const allureAttachmentSchema = z.object({ + name: z.string(), + source: z.string(), + type: z.string(), +}) + +const allureLabelSchema = z.object({ + name: z.string(), + value: z.string(), +}) + +const allureParameterSchema = z.object({ + name: z.string(), + value: z.string(), + excluded: z.boolean().optional(), + mode: z.enum(['default', 'masked', 'hidden']).optional(), +}) + +const allureLinkSchema = z.object({ + name: z.string().optional(), + url: z.string(), + type: z.string().optional(), +}) + +const allureResultSchema = z.object({ + name: z.string(), + status: allureStatusSchema, + uuid: z.string(), + start: z.number(), + stop: z.number(), + fullName: z.string().optional(), + historyId: z.string().optional(), + testCaseId: z.string().optional(), + description: z.string().optional(), + descriptionHtml: z.string().optional(), + stage: z.string().optional(), + statusDetails: allureStatusDetailsSchema, + attachments: allureAttachmentSchema.array().nullable().optional(), + labels: allureLabelSchema.array().nullable().optional(), + links: allureLinkSchema.array().nullable().optional(), + parameters: allureParameterSchema.array().nullable().optional(), + steps: z.any().nullable().optional(), +}) + +type AllureResult = z.infer + +const mapAllureStatus = (status: AllureResult['status']): ResultStatus => { + switch (status) { + case 'passed': + return 'passed' + case 'failed': + return 'failed' + case 'broken': + return 'blocked' + case 'skipped': + return 'skipped' + case 'unknown': + return 'passed' + } +} + +const getFolderFromLabels = (labels: AllureResult['labels']): string => { + if (!labels || labels.length === 0) return '' + + const labelMap = new Map() + for (const label of labels) { + if (!labelMap.has(label.name)) { + labelMap.set(label.name, label.value) + } + } + + return ( + labelMap.get('suite') || + labelMap.get('parentSuite') || + labelMap.get('feature') || + labelMap.get('package') || + '' + ) +} + +const getTCaseMarkerFromLinks = (links: AllureResult['links']): string | undefined => { + if (!links || links.length === 0) return undefined + + for (const link of links) { + if (link.type !== 'tms') continue + + // Try parsing as QA Sphere URL first + const parsed = parseTCaseUrl(link.url) + if (parsed) { + return getTCaseMarker(parsed.project, parsed.tcaseSeq) + } + + // Fall back to regex on link name + if (link.name) { + const match = link.name.match(/\b[A-Z]+-\d+\b/) + if (match) { + return match[0] + } + } + } + + return undefined +} + +const buildMessage = ( + result: AllureResult, + status: ResultStatus, + options: ParserOptions +): string => { + let message = '' + const details = result.statusDetails + + if (!details) return message + + const includeMessage = !(status === 'passed' && options.skipStdout === 'on-success') + const includeTrace = !(status === 'passed' && options.skipStderr === 'on-success') + + if (includeMessage && details.message) { + message += `

Message:

${escapeHtml(details.message)}
` + } + + if (includeTrace && details.trace) { + message += `

Trace:

${escapeHtml(details.trace)}
` + } + + return message +} + +/** + * Parses Allure results from a directory of JSON result files. + * + * @param data - The directory path containing Allure result files (not file content). + * For Allure, the Parser's `data` parameter is a directory path since + * Allure results are spread across multiple files in a directory. + * @param attachmentBaseDirectory - Same as `data` for Allure (the results directory). + * @param options - Parser options for controlling output inclusion. + */ +export const parseAllureResults: Parser = async ( + data: string, + attachmentBaseDirectory: string, + options: ParserOptions +): Promise => { + const dirPath = data + const entries = readdirSync(dirPath) + const resultFiles = entries.filter((f) => f.endsWith('-result.json')) + + const testcases: TestCaseResult[] = [] + const attachmentsPromises: Array<{ + index: number + promise: Promise + }> = [] + + for (const file of resultFiles) { + const filePath = path.join(dirPath, file) + let raw: string + try { + raw = readFileSync(filePath, 'utf8') + } catch { + console.warn(`Warning: Could not read file ${filePath}, skipping`) + continue + } + + let json: unknown + try { + json = JSON.parse(raw) + } catch { + console.warn(`Warning: Malformed JSON in ${filePath}, skipping`) + continue + } + + const parsed = allureResultSchema.safeParse(json) + if (!parsed.success) { + console.warn(`Warning: Invalid Allure result in ${filePath}, skipping`) + continue + } + + const result = parsed.data + const status = mapAllureStatus(result.status) + const folder = getFolderFromLabels(result.labels) + const duration = result.stop - result.start + + // Extract test case marker: TMS links > test name + const markerFromLinks = getTCaseMarkerFromLinks(result.links) + const name = markerFromLinks ? `${markerFromLinks}: ${result.name}` : result.name + + const numTestcases = testcases.push({ + name, + folder, + status, + message: buildMessage(result, status, options), + timeTaken: duration, + attachments: [], + }) + + // Collect attachment file paths + const attachmentPaths: string[] = [] + if (result.attachments) { + for (const attachment of result.attachments) { + attachmentPaths.push(attachment.source) + } + } + + if (attachmentPaths.length > 0) { + attachmentsPromises.push({ + index: numTestcases - 1, + promise: getAttachments(attachmentPaths, attachmentBaseDirectory), + }) + } + } + + // Resolve all attachment promises + const attachments = await Promise.all(attachmentsPromises.map((p) => p.promise)) + attachments.forEach((tcaseAttachments, i) => { + const tcaseIndex = attachmentsPromises[i].index + testcases[tcaseIndex].attachments = tcaseAttachments + }) + + return testcases +}