diff --git a/package-lock.json b/package-lock.json index cd5693c8..0e7e6aec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@jrpool/kilotest", - "version": "57.2.3", + "version": "58.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@jrpool/kilotest", - "version": "57.2.3", + "version": "58.0.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "*", @@ -5525,9 +5525,9 @@ } }, "node_modules/hono": { - "version": "4.13.12", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.12.tgz", - "integrity": "sha512-6E2QDAc9Ick9Sq77ZrGS/dk2WUYni91aufTw6LJKpV7w8kW5/GxVUc650FOADOmlwg3K+f7Pun6XlV+pYmW6gw==", + "version": "4.13.13", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.13.tgz", + "integrity": "sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -6436,9 +6436,9 @@ } }, "node_modules/mdast-util-find-and-replace": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", - "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.3.tgz", + "integrity": "sha512-xPgpDNl0/OXHsI7TlaIs22lWnH3KlpvZdXG1ET/m/YT2Hhdkx8lJV4kLphE6l9simyM4KPDFBwas08BgEob6Jw==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 9899e866..7ed966e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@jrpool/kilotest", - "version": "57.3.0", + "version": "58.0.0", "description": "Ensemble testing for front-end web quality", "type": "module", "main": "index.ts", diff --git a/test/fixtures/buildFixtures.ts b/test/fixtures/buildFixtures.ts index 9711afd0..22f93e8c 100644 --- a/test/fixtures/buildFixtures.ts +++ b/test/fixtures/buildFixtures.ts @@ -203,11 +203,20 @@ const main = async () => { const preventedCatalog = { '0': catalogItem('IMG', 'An image', '/html/body/img[1]', '0:0:200:100') }; + // As in real reports, the prevented engine has a test act, with no standard result. + const preventedAlfaAct = { + type: 'test', + which: engineAlfa, + startTime: '26-01-01T00:00', + data: {prevented: true, error: 'page timed out'}, + endTime: '26-01-01T00:01' + }; const preventedActs = [ testAct(engineAxe, [ instance('r11', 'The image has no alt text', 'failed', issueLinkNoText, 0, 2, 1) - ]) + ]), + preventedAlfaAct ]; const preventedReport = report('260101T0006-prv', 'Prevented Page', 'https://example.com/prevented', preventedActs, preventedCatalog); diff --git a/test/fixtures/db/reports/260101T0006-prv.json b/test/fixtures/db/reports/260101T0006-prv.json index 5ad49b28..90ba0c43 100644 --- a/test/fixtures/db/reports/260101T0006-prv.json +++ b/test/fixtures/db/reports/260101T0006-prv.json @@ -42,6 +42,16 @@ } } } + }, + { + "type": "test", + "which": "alfa", + "startTime": "26-01-01T00:00", + "data": { + "prevented": true, + "error": "page timed out" + }, + "endTime": "26-01-01T00:01" } ], "jobData": { diff --git a/util.ts b/util.ts index 246f3056..15d18b95 100644 --- a/util.ts +++ b/util.ts @@ -122,6 +122,7 @@ export type AnnotatedAct = Act & { // and the else/after-return branch is narrowed to UsableReport. export type UsableReport = Omit & { error?: never; + id: string; target: {what: string; url: string}; catalog: Catalog; jobData: NonNullable & {endTime: string; issuelessRules?: string[]}; diff --git a/web/listDiagnoses/index.html b/web/listDiagnoses/index.html index 0991b2fd..606f2eda 100644 --- a/web/listDiagnoses/index.html +++ b/web/listDiagnoses/index.html @@ -14,28 +14,15 @@

Kilotest: Diagnoses of __issue__ violation by HTML element __catalogIndex__ on __target__ page

Basics

-

About the __target__ page

-
    -
  • URL: __urlLink__
    • -
  • __testInfo__
    • -
-

About HTML element __catalogIndex__

-
    -
  • Tag name: __tagName__
    • -
  • Text: __text__
    • -
  • Start tag: __startTag__
    • -
  • XPath: __pathID__
    • -
  • Bounding box: __box__
    • -
+

About the page

+__pageFacts__ +

About the issue

+__issueFacts__ +

About the violator

+__violatorFacts__
    __takeMeThere__
-

About the __issue__ issue

-
    -
  • Why it matters: __why__
    • -
  • Priority: __priority__
    • -
  • Related WCAG standard: __wcag__
    • -

Diagnoses

Here is how rule engines diagnose the __issue__ issue for HTML element __catalogIndex__ of the __target__ page.

    diff --git a/web/listDiagnoses/index.ts b/web/listDiagnoses/index.ts index 25cf7c75..92ed705d 100644 --- a/web/listDiagnoses/index.ts +++ b/web/listDiagnoses/index.ts @@ -14,10 +14,12 @@ import { ruleEngines } from '../../util.ts'; import { + getIssueFactsLines, getPageDataStrings, + getPageFactsLines, + getReportData, getTextFragmentHref, - getWCAGLink, - getWeightName + getViolatorFactsLines } from '../util.ts'; import {issues as issueSpecs} from 'testaro-issues'; @@ -32,37 +34,33 @@ const populateQuery = async ( pathID: string | null, query: Record ) => { - // Get descriptions of the page facts. - const pageDataStrings = await getPageDataStrings(timeStamp, jobID); + // Get the report. + const report = await getReport(timeStamp, jobID); // If this failed: - if (pageDataStrings.error !== undefined) { + if (isReportError(report)) { // Populate the query with the reason. - query.error = pageDataStrings.error; + query.error = report.error; // Stop populating the query. return; } - const {testInfo, url, urlLink, description} = pageDataStrings; - // Otherwise, i.e. if it succeeded, get the report. - const report = await getReport(timeStamp, jobID); + // Otherwise, i.e. if it succeeded, get descriptions of the page facts. + const pageDataStrings = getPageDataStrings(report); // If this failed: - if (isReportError(report)) { + if (pageDataStrings.error !== undefined) { // Populate the query with the reason. - query.error = report.error; + query.error = pageDataStrings.error; // Stop populating the query. return; } + const {testInfo, url, description} = pageDataStrings; const {catalog} = report; - // Otherwise, i.e. if it succeeded, get the catalog item of the specified violator. - const catalogItem = catalog[catalogIndex]; - const boxID = catalogItem?.boxID; - const startTag = catalogItem?.startTag; - const tagName = catalogItem?.tagName; - const text = catalogItem?.text; query.catalogIndex = catalogIndex; const lines: string[] = []; const margin = ' '.repeat(6); + const catalogItem = catalog[catalogIndex]; + // If the violator has a linkable text item, add a take-me-there link. if (catalogIndex && catalogItem?.textLinkable) { - const href = getTextFragmentHref(text as string, url); + const href = getTextFragmentHref(catalogItem.text as string, url); const label = `Take me to element ${catalogIndex} on the page (in a new tab)`; const link = `Take me there`; query.takeMeThere = `${margin}

    ${link}

    `; @@ -70,10 +68,10 @@ const populateQuery = async ( else { query.takeMeThere = ''; } - // Add facts about the issue to the query. + // Add facts about the page to the query. query.target = description; - query.urlLink = urlLink; query.testInfo = testInfo; + query.pageFacts = getPageFactsLines(pageDataStrings, getReportData(report), margin).join('\n'); query.issue = issueSpecs[issueID]?.summary; // If adding the issue summary failed: if (!query.issue) { @@ -82,29 +80,9 @@ const populateQuery = async ( // Stop populating the query. return; } - // Otherwise, i.e. if it succeeded, get the issue details. - const issue = issueSpecs[issueID]!; - const {wcag, weight, why} = issue; - query.why = why; - query.priority = getWeightName(weight); - query.wcag = `${wcag}`; - query.tagName = tagName || 'HTML'; - if (text && !['HTML', 'BODY', 'HEAD', 'SCRIPT', 'STYLE', 'NOSCRIPT'].includes(tagName as string)) { - const textString = text.split('\n').join(' … '); - query.text = `${htmlSafe(textString)}`; - } - else { - query.text = '[not applicable]'; - } - query.startTag = htmlSafe(startTag ?? '') || '[not obtained]'; - query.pathID = pathID || '[not obtained]'; - if (boxID) { - const dims = boxID.split(':'); - query.box = `x = ${dims[0]}, y = ${dims[1]}, width = ${dims[2]}, height = ${dims[3]}`; - } - else { - query.box = '[not obtained]'; - } + // Otherwise, i.e. if it succeeded, add the issue-facts and violator-facts lists to the query. + query.issueFacts = getIssueFactsLines(report, issueID, margin).join('\n'); + query.violatorFacts = getViolatorFactsLines(report, issueID, catalogIndex, pathID, margin).join('\n'); // Initialize an array of diagnoses. const diagnoses: any[] = []; // For each violating standard instance that pertains to this combination of issue and violator: diff --git a/web/listDiagnoses/listDiagnoses.test.ts b/web/listDiagnoses/listDiagnoses.test.ts index f68409a1..7529cfd8 100644 --- a/web/listDiagnoses/listDiagnoses.test.ts +++ b/web/listDiagnoses/listDiagnoses.test.ts @@ -33,7 +33,7 @@ mock.module('../../util.ts', { mock.module('../util.ts', { exports: { ...realWebUtil, - getPageDataStrings: async (...args: any[]) => { + getPageDataStrings: (...args: any[]) => { if (pageDataStringsOverride !== null) { return pageDataStringsOverride; } @@ -108,7 +108,7 @@ test('listDiagnoses returns ok with no diagnoses for a nonexistent catalog index assert.ok(!diagnosisTexts.some(t => t.includes('The link does not have an accessible name'))); }); -test('listDiagnoses returns an error when getReport fails after getPageDataStrings succeeds', async () => { +test('listDiagnoses returns an error when getReport fails', async () => { getReportCallCount = 0; failGetReportOnCall = 1; try { @@ -121,6 +121,18 @@ test('listDiagnoses returns an error when getReport fails after getPageDataStrin } }); +test('listDiagnoses returns an error when getPageDataStrings fails after getReport succeeds', async () => { + pageDataStringsOverride = {error: 'Page data strings error'}; + try { + const result = await answer('linkNoText/260101T0000/mix/0'); + assert.equal(result.status, 'error'); + assert.equal(result.message, 'Page data strings error'); + } + finally { + pageDataStringsOverride = null; + } +}); + test('listDiagnoses includes a take-me-there link for a text-linkable catalog item', async () => { const dbDir = (await import('../../test/dbFixture.ts')).fixtureDBDir; const reportsDir = path.join(dbDir, 'reports'); diff --git a/web/listIssues/index.html b/web/listIssues/index.html index 2d8a16bc..b35efb62 100644 --- a/web/listIssues/index.html +++ b/web/listIssues/index.html @@ -13,20 +13,10 @@

    Kilotest: Issues reported for __target__

    -

    About the __target__ page

    -
      -
    • URL: __urlLink__
      • -
    • __testInfo__
      • - __preventions__ -
    +

    About the page

    +__pageFacts__

    Should Kilotest retest the page?

    Issues reported

    -

    Summary

    -

    __reporterCount__ reported issues (__reporters__)

    -

    __issueCount__ reported (__highestCount__ highest, __highCount__ high, __lowCount__ low, __lowestCount__ lowest priority)

    -

    __violatorCount__ reported

    -

    Download the full technical report

    -

    Details

    Highest priority: __highestCount__
      diff --git a/web/listIssues/index.ts b/web/listIssues/index.ts index ccf47a9d..4190dbd9 100644 --- a/web/listIssues/index.ts +++ b/web/listIssues/index.ts @@ -8,147 +8,132 @@ import { getReport, getTestActInstances, - htmlSafe, - isUsableReport, + isReportError, objectSort, - populateTemplate, - ruleEngines + populateTemplate } from '../../util.ts'; +import type {UsableReport} from '../../util.ts'; import { - getPageData, + getIssueFactsLines, getPageDataStrings, + getPageFactsLines, + getEngineIDs, getEngineNamesString, - getWCAGLink, getWeightName } from '../util.ts'; +import type {ResultsSummary} from '../util.ts'; import {issues as issueSpecs} from 'testaro-issues'; // FUNCTIONS // Returns data on the issues reported by a report. -const getIssuesData = async (timeStamp: string, jobID: string) => { - // Get the report. - const report = await getReport(timeStamp, jobID); - // If it exists and is valid: - if (isUsableReport(report)) { - // Initialize the temporary data. - const temp = { - issues: {} as Record, - reporters: new Set(), - violators: new Set() - }; - // Initialize the final data. - const final: Record = { - reporters: [], - reporterList: '', - reporterCount: 0, - violatorCount: 0, - preventions: report.jobData.preventions, - issues: { - 4: [], - 3: [], - 2: [], - 1: [] - }, - issueCount: 0 - }; - // For each violating standard instance of each test act: - getTestActInstances(report, {violationsOnly: true}).forEach(({act, instance}) => { - const {catalogIndex, issueID} = instance; - const which = act.which!; - // If it identifies a non-ignorable issue: - if (issueID && issueID !== 'ignorable') { - const issueClassification = issueSpecs[issueID]; - // If the issue has a current weighted classification: - if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) { - const {summary, wcag, weight, why} = issueClassification; - // Initialize the temporary data on the issue if necessary. - temp.issues[issueID] ??= { - issueID, - summary, - wcag, - why, - weight, - reporters: new Set(), - reporterList: '', - violators: new Set() - }; - // Ensure the rule engine is in the temporary data. - temp.issues[issueID].reporters.add(which); - temp.reporters.add(which); - // If the instance has a catalog index: - if (catalogIndex) { - // Ensure the violator is in the temporary data. - temp.issues[issueID].violators.add(catalogIndex); - temp.violators.add(String(catalogIndex)); - } +const getIssuesData = (report: UsableReport): ResultsSummary & Record => { + // Initialize the temporary data. + const temp = { + issues: {} as Record, + reporters: new Set(), + violators: new Set() + }; + // Initialize the final data. + const final: ResultsSummary & Record = { + reporters: [], + reporterList: '', + reporterCount: 0, + engineCount: 0, + testedEngineCount: 0, + violationCount: 0, + violatorCount: 0, + issues: { + 4: [], + 3: [], + 2: [], + 1: [] + }, + issueCount: 0 + }; + // For each violating standard instance of each test act: + getTestActInstances(report, {violationsOnly: true}).forEach(({act, instance}) => { + const {catalogIndex, issueID} = instance; + const which = act.which!; + // If it identifies a non-ignorable issue: + if (issueID && issueID !== 'ignorable') { + const issueClassification = issueSpecs[issueID]; + // If the issue has a current weighted classification: + if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) { + const {summary, wcag, weight, why} = issueClassification; + // Initialize the temporary data on the issue if necessary. + temp.issues[issueID] ??= { + issueID, + summary, + wcag, + why, + weight, + reporters: new Set(), + reporterList: '', + violationCount: 0, + violators: new Set() + }; + // Ensure the rule engine is in the temporary data. + temp.issues[issueID].reporters.add(which); + temp.reporters.add(which); + // Increment the violation counts. + temp.issues[issueID].violationCount++; + final.violationCount++; + // If the instance has a catalog index: + if (catalogIndex) { + // Ensure the violator is in the temporary data. + temp.issues[issueID].violators.add(catalogIndex); + temp.violators.add(String(catalogIndex)); } } - }); - // Finish populating the final data. - final.reporterList = getEngineNamesString(temp.reporters); - final.reporterCount = temp.reporters.size; - final.violatorCount = temp.violators.size; - Object.values(temp.issues).forEach(issue => { - const {issueID, summary, wcag, why, weight} = issue; - const finalIssue: Record = { - issueID, - summary, - wcag, - why, - weight - }; - finalIssue.reporterList = getEngineNamesString(issue.reporters); - finalIssue.reporterCount = issue.reporters.size; - finalIssue.violatorCount = issue.violators.size; - final.issues[issue.weight].push(finalIssue); - }); - final.issueCount = Object.keys(temp.issues).length; - // For each weight: - [4, 3, 2, 1].forEach(weight => { - // Sort its issues in the final data alphabetically by reporter names. - objectSort(final.issues[weight], 'reporterList', 'alpha'); - // Sort the issues again in descending reporter-count order, making this the primary order. - objectSort(final.issues[weight], 'reporterCount', 'numericDown'); - }); - // Return the data. - return final; - } - // Otherwise, i.e. if it is invalid or does not exist, return this. - return {error: 'Report missing or invalid.'}; -}; -// Get page and issues data from a report. -const getData = async (timeStamp: string, jobID: string) => { - const pageData = await getPageData(timeStamp, jobID); - const issuesData: any = await getIssuesData(timeStamp, jobID); - const pageError = ('error' in pageData ? pageData.error : ''); - const issuesError = issuesData.error || ''; - const errors = [pageError, issuesError].filter(Boolean).join('; '); - // If the data of either type are missing or invalid: - if (errors) { - // Return this. - return {error: errors}; - } - // Otherwise, return the data. - return { - pageData, - issuesData - }; + } + }); + // Finish populating the final data. + const {calledIDs, preventedIDs} = getEngineIDs(report); + final.engineCount = calledIDs.length; + final.testedEngineCount = calledIDs.length - preventedIDs.length; + final.reporterList = getEngineNamesString(temp.reporters); + final.reporterCount = temp.reporters.size; + final.violatorCount = temp.violators.size; + Object.values(temp.issues).forEach(issue => { + const {issueID, summary, wcag, why, weight} = issue; + const finalIssue: Record = { + issueID, + summary, + wcag, + why, + weight + }; + finalIssue.reporterList = getEngineNamesString(issue.reporters); + finalIssue.reporterCount = issue.reporters.size; + finalIssue.violationCount = issue.violationCount; + finalIssue.violatorCount = issue.violators.size; + final.issues[issue.weight].push(finalIssue); + }); + final.issueCount = Object.keys(temp.issues).length; + // For each weight: + [4, 3, 2, 1].forEach(weight => { + // Sort its issues in the final data alphabetically by reporter names. + objectSort(final.issues[weight], 'reporterList', 'alpha'); + // Sort the issues again in descending reporter-count order, making this the primary order. + objectSort(final.issues[weight], 'reporterCount', 'numericDown'); + }); + // Return the data. + return final; }; // Adds parameters to a query for the answer page. const populateQuery = async (timeStamp: string, jobID: string, query: Record) => { - // Get data on the target and its issues according to the report. - const data: any = await getData(timeStamp, jobID); - const {pageData, issuesData} = data; + // Get the report. + const report = await getReport(timeStamp, jobID); // If this failed: - if (data.error) { + if (isReportError(report)) { // Populate the query with the reason. - query.error = data.error; + query.error = report.error; // Stop populating the query. return; } // Otherwise, i.e. if it succeeded, get fact descriptions for the target. - const pageInfo = await getPageDataStrings(timeStamp, jobID, pageData); + const pageInfo = getPageDataStrings(report); // If this failed: if (pageInfo.error !== undefined) { // Populate the query with the reason. @@ -156,45 +141,24 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record { - const [engineName, engineSponsor] = ruleEngines[preventedEngineID] ?? [preventedEngineID, 'unknown sponsor']; - const engineNameString = `${engineName} (${engineSponsor})`; - const causeString = htmlSafe(preventions[preventedEngineID]); - const preventionString = `${margin}
    • Page not testable by ${engineNameString}: ${causeString}
      • `; - preventionStrings.push(preventionString); - }); - // Add prevention notices to the query. - query.preventions = preventionStrings.join('\n'); + // Add the page-facts list to the query. + query.pageFacts = getPageFactsLines(pageInfo, issuesData, margin).join('\n'); + const {issues} = issuesData; // Add report data to the query. query.timeStamp = timeStamp; query.jobID = jobID; - // Add reporter information to the query. - query.reporterCount = reporterCount === 1 ? '1 rule engine' : `${reporterCount} rule engines`; - query.reporters = reporterList; // Add a summary of the issues to the query. - query.issueCount = issueCount === 1 ? '1 issue was' : `${issueCount} issues were`; query.highestCount = issues[4].length; query.highCount = issues[3].length; query.lowCount = issues[2].length; query.lowestCount = issues[1].length; - // Add a violator count to the query. - query.violatorCount = violatorCount === 1 ? '1 violator was' : `${violatorCount} violators were`; // For each weight: [4, 3, 2, 1].forEach(weight => { const weightName = getWeightName(weight); @@ -208,35 +172,15 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record${wcag}`; + const {issueID, summary} = issueData; // Add the start of a list item to the lines. detailsLines.push(`${margin}
    • `); // Add a heading summarizing the issue to the lines. detailsLines.push(`${margin}
      ${summary}
      `); - // Add the start of a fact list about the issue to the lines. - detailsLines.push(`${margin}
        `); // Add the issue facts to the lines. - detailsLines.push(`${margin}
      • Why it matters: ${why}`); - detailsLines.push(`${margin}
      • Related WCAG standard: ${wcagLink}`); - const reporterCountString = reporterCount === 1 ? '1 rule engine' : `${reporterCount} rule engines`; detailsLines.push( - `${margin}
      • Reported by ${reporterCountString} (${reporterList})
        • ` + ...getIssueFactsLines(report, issueID, `${margin} `, 'pseudoTopLevel') ); - const violatorCountString = violatorCount === 1 - ? '1 violator was' - : `${violatorCount} violators were`; - detailsLines.push(`${margin}
      • ${violatorCountString} reported
        • `); - // Add the end of the fact list to the lines. - detailsLines.push(`${margin}
      `); // Add the start of a link list to the lines. detailsLines.push(`${margin}