fix: preserve partial benchmark failures in published results

.github/workflows/benchmark.yaml

@@ -240,8 +240,8 @@ jobs:
         id: registry-raw
         # Any non-zero vlt exit code is a potential outage or misconfiguration.
         # This must run before `bench process`, which rewrites exit codes in
-        # place (dropping failed runs), so raw per-run failures are only
-        # visible here.
+        # place. original_exit_codes also preserves those outcomes in the
+        # processed output, but validation reads the raw export here.
         if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.partial_run != 'true'
         uses: actions/github-script@v9
         with:
@@ -335,12 +335,11 @@ jobs:
               if (!match) continue
               const label = `${match[1]} ${match[2]}`
 
-              // Registries where every run failed are stamped with a non-zero
-              // exit code and a zero mean by clean-benchmarks and rendered as
-              // failures on the site, so they are excluded from the speed
-              // comparison. A failed vlt is already reported by the raw scan.
+              // Compare only complete results. Partial survivor statistics do
+              // not establish a fair speed comparison. Failures are already
+              // reported by the raw scan before processing.
               const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? [])
-                .filter((r) => r.mean > 0 && (r.exit_codes ?? []).every((c) => c === 0))
+                .filter((r) => r.mean > 0 && r.status !== 'partial' && !(r.dropped_runs > 0) && (r.exit_codes ?? []).every((c) => c === 0))
               const vlt = ok.find((r) => r.command === 'vlt')
               if (!vlt) continue
 

.github/workflows/test.yml

@@ -0,0 +1,24 @@
+name: Data and App Tests
+on:
+  pull_request:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          package-manager-cache: false
+      - run: npm install --ignore-scripts --no-package-lock
+        working-directory: app
+      - run: node --test scripts/*.test.js
+      - name: Test app data semantics
+        run: |
+          if [ -d app/tests ]; then node --test app/tests/*.test.*; fi
+      - run: npm run build
+        working-directory: app

README.md

@@ -176,7 +176,7 @@ This suite also tests the performance of basic script execution (ex. `npm run fo
 - 4. Process benchmark output:
 
   ```bash
-  # Filter failed runs and generate dated + latest outputs and chart data
+  # Record run outcomes, filter failed timings, and generate dated + latest output
   ./bench process
   # Or pass --process directly when running
   ./bench run --fixtures=next --runs=3 --process
@@ -199,6 +199,35 @@ The workflow:
 
 ## Results
 
+### Failed and partial runs
+
+Processed benchmark results retain `original_exit_codes`, `attempted_runs`,
+`successful_runs`, `dropped_runs`, and a `status` of `success`, `partial`, or
+`failure`. These counts refer to measured runs, excluding Hyperfine warmups.
+Cleaning the same output again preserves the original counts and exit codes.
+The existing `times` and `exit_codes` arrays contain successful runs; total
+failure keeps the existing zero-time DNF sentinel for compatibility.
+
+Partial timing statistics describe the successful runs only. A single survivor
+has `stddev: null` because it cannot establish variability. CPU `user` and
+`system` fields are omitted whenever runs were dropped: Hyperfine exports only
+aggregate CPU values, which cannot be recalculated for the surviving sample.
+
+Chart data adds optional `<command>_partial`, `<command>_attempted_runs`,
+`<command>_successful_runs`, and `<command>_dropped_runs` fields in both total and
+per-package datasets. Tables and chart notices label partial results with their
+success counts. Synthetic averages retain that warning. Commands with any
+partial result in the selected comparisons are excluded from rankings; history
+omits that command's affected daily variation and category average.
+
+This is an additive schema change. Historical JSON without these fields remains
+readable, but its completeness is unknown: failed attempts discarded by older
+processing cannot be reconstructed. Regenerate from raw benchmark artifacts to
+recover counts when those artifacts are still available. CI checks that inspect
+raw `exit_codes` must run before `./bench process`; the registry failure scan
+already does so. Persisted original codes also remain available in dated result
+files on gh-pages.
+
 ### Console Output
 
 Each benchmark run provides a summary in the console:

app/package.json

@@ -4,6 +4,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
+    "test": "node --test tests/*.test.js",
     "dev": "vite",
     "format": "prettier --write . --log-level warn --ignore-path ./.prettierignore --cache",
     "build": "tsc -b && vite build",

app/src/components/history-chart.tsx

@@ -237,6 +237,10 @@ export const HistoryChart = ({
       </div>
 
       <div className="bg-card rounded-xl p-3 md:p-6 border-border border-[1px] overflow-hidden">
+        <p className="mb-4 text-xs text-muted-foreground">
+          Days with partial results are omitted for that command and variation.
+          Older results without run counts have unknown completeness.
+        </p>
         <ChartContainer
           config={chartConfig}
           className="min-h-[180px] md:min-h-[250px] w-full"

app/src/components/ui/chart.tsx

@@ -2,6 +2,7 @@ import * as React from "react";
 import * as RechartsPrimitive from "recharts";
 
 import { cn } from "@/lib/utils";
+import { partialResultLabel } from "@/lib/run-completeness";
 
 /**
  * Creates a CSS repeating-linear-gradient that matches the SVG diagonal-stripe
@@ -290,6 +291,10 @@ function ChartTooltipContent({
           const key = `${nameKey || item.name || item.dataKey || "value"}`;
           const itemConfig = getPayloadConfigFromPayload(config, item, key);
           const isDnf = isDnfPayload(item);
+          const partial = partialResultLabel(
+            item.payload,
+            String(item.dataKey ?? ""),
+          );
           const dnfIndicatorColor = isDnf ? getDnfIndicatorColor(item) : null;
           // Resolve the indicator color. For DNF entries, getDnfIndicatorColor
           // reads the hex color from `${dataKey}_fill` in the payload. For
@@ -366,6 +371,11 @@ function ChartTooltipContent({
                       <span className="text-muted-foreground">
                         {itemConfig?.label || item.name}
                       </span>
+                      {partial && (
+                        <span className="text-amber-700 dark:text-amber-400">
+                          {partial}
+                        </span>
+                      )}
                     </div>
                     {(item.value !== undefined || isDnf) && (
                       <span className="text-foreground font-mono font-medium tabular-nums">

app/src/components/variation/chart.tsx

@@ -33,6 +33,7 @@ import { CHART_DEFAULTS } from "@/constants";
 import { formatPackageManagerLabel, getFixtureId } from "@/lib/utils";
 import { getFrameworkIcon } from "@/lib/get-icons";
 import { useMediaQuery } from "@/hooks/use-media-query";
+import { partialResultLabel } from "@/lib/run-completeness";
 
 import type { ChartConfig } from "@/components/ui/chart";
 import type {
@@ -126,6 +127,7 @@ const HorizontalBarTooltipContent = ({
               item.color;
 
           const isBaseline = item.payload?.isBaseline === true;
+          const partial = partialResultLabel(item.payload);
 
           return (
             <div key={index} className="flex items-center gap-2">
@@ -140,6 +142,11 @@ const HorizontalBarTooltipContent = ({
               <div className="flex flex-1 justify-between items-center gap-4">
                 <span className="text-muted-foreground">
                   {item.name ?? item.dataKey}
+                  {partial && (
+                    <span className="block text-amber-700 dark:text-amber-400">
+                      {partial}
+                    </span>
+                  )}
                   {isBaseline && (
                     <span className="ml-1 text-[10px] italic opacity-70">
                       baseline
@@ -323,6 +330,24 @@ export const VariationChart = ({
 
   // Variation data already respects fixture filters upstream
   const filteredVariationData = variationData;
+  const partialResults = filteredVariationData.flatMap((row) =>
+    filteredPackageManagers.flatMap((pm) => {
+      const label = partialResultLabel({ ...row }, pm);
+      return label ? [`${row.fixture} / ${pm}: ${label}`] : [];
+    }),
+  );
+  const partialNotice = partialResults.length > 0 && (
+    <div className="rounded-md border border-amber-500 p-3 text-sm text-amber-800 dark:text-amber-300">
+      <p>
+        Partial results include successful runs only and may appear faster
+        because failed or timed-out runs were dropped. Commands with partial
+        results are excluded from rankings and history averages.
+      </p>
+      <ul className="mt-2 list-disc pl-5">
+        {partialResults.map((label) => <li key={label}>{label}</li>)}
+      </ul>
+    </div>
+  );
 
   const variationActivePackageManagers = useMemo(() => {
     const active = new Set<PackageManager>();
@@ -372,7 +397,7 @@ export const VariationChart = ({
   // Always compute both data structures to avoid conditional hook calls
   const consolidatedData = useMemo(() => {
     return filteredVariationData.map((item): ConsolidatedChartItem => {
-      const chartItem: ConsolidatedChartItem = { fixture: item.fixture };
+      const chartItem: ConsolidatedChartItem = { ...item, fixture: item.fixture };
       const slowest = fixtureSlowestValues.get(item.fixture);
 
       filteredPackageManagers.forEach((pm) => {
@@ -430,6 +455,7 @@ export const VariationChart = ({
     // Task runners & registries: horizontal bar charts per fixture, sorted by speed
     return (
       <div className="space-y-8">
+        {partialNotice}
         <div className="flex flex-col gap-3 md:gap-0 md:flex-row items-start md:items-center justify-between">
           <h3 className="text-base md:text-lg w-full font-medium tracking-tighter flex items-center gap-2 group">
             <Clock className="text-muted-foreground flex-shrink-0" />
@@ -491,6 +517,9 @@ export const VariationChart = ({
                     ? getDnfPatternFill(fixtureId, pm)
                     : getColor(pm),
                   dnf: isDnf,
+                  partial: fixtureResult[`${pm}_partial`] === true,
+                  attempted_runs: fixtureResult[`${pm}_attempted_runs`],
+                  successful_runs: fixtureResult[`${pm}_successful_runs`],
                   dnfColor: getColor(pm),
                   isBaseline: isBaselinePackageManager(pm, isRegistry),
                 };
@@ -684,6 +713,7 @@ export const VariationChart = ({
     if (isMobile) {
       return (
         <div className="space-y-6">
+          {partialNotice}
           {consolidatedHeader}
 
           {/* Legend */}
@@ -752,6 +782,9 @@ export const VariationChart = ({
                       ? getDnfPatternFill(fixtureId, pm)
                       : getColor(pm),
                     dnf: isDnf,
+                    partial: fixtureResult[`${pm}_partial`] === true,
+                    attempted_runs: fixtureResult[`${pm}_attempted_runs`],
+                    successful_runs: fixtureResult[`${pm}_successful_runs`],
                     dnfColor: getColor(pm),
                     pm,
                   };
@@ -836,6 +869,7 @@ export const VariationChart = ({
     // DESKTOP: Grouped bar chart (existing layout)
     return (
       <div className="space-y-8">
+        {partialNotice}
         {consolidatedHeader}
 
         <div className="bg-card rounded-xl p-3 md:p-6 border-border border-[1px] overflow-hidden">
@@ -956,6 +990,7 @@ export const VariationChart = ({
   // Uses horizontal bar charts sorted fastest-to-slowest for all screen sizes
   return (
     <div className="space-y-8">
+      {partialNotice}
       <div>
         <h3 className="text-lg font-medium tracking-tighter flex items-center gap-2 group">
           <StopWatch className="text-muted-foreground" />
@@ -1000,6 +1035,9 @@ export const VariationChart = ({
                 value: resolvedValue,
                 fill: isDnf ? getDnfPatternFill(fixtureId, pm) : fillColor,
                 dnf: isDnf,
+                partial: fixtureResult[`${pm}_partial`] === true,
+                attempted_runs: fixtureResult[`${pm}_attempted_runs`],
+                successful_runs: fixtureResult[`${pm}_successful_runs`],
                 dnfColor: getColor(pm),
               };
             })

app/src/components/variation/table.tsx

@@ -28,6 +28,7 @@ import {
 import { ShareButton } from "@/components/share-button";
 import { usePackageManagerFilter } from "@/contexts/package-manager-filter-context";
 import { Clock, StopWatch } from "@/components/icons";
+import { partialResultLabel } from "@/lib/run-completeness";
 
 import type {
   BenchmarkChartData,
@@ -53,6 +54,7 @@ interface TransposedVariationRow {
   packageManager: PackageManager;
   fixtureValues: Partial<Record<Fixture, number>>;
   fixtureDnf: Partial<Record<Fixture, boolean>>;
+  fixturePartial: Partial<Record<Fixture, string>>;
 }
 
 const columnHelper = createColumnHelper<TransposedVariationRow>();
@@ -91,12 +93,17 @@ export const VariationTable = ({
       filteredPackageManagers.map((packageManager) => {
         const fixtureValues: Partial<Record<Fixture, number>> = {};
         const fixtureDnf: Partial<Record<Fixture, boolean>> = {};
+        const fixturePartial: Partial<Record<Fixture, string>> = {};
 
         variationData.forEach((fixtureResult) => {
           const fixture = fixtureResult.fixture;
           const dnfKey = `${packageManager}_dnf` as keyof FixtureResult;
           const value = fixtureResult[packageManager];
           const isDnf = fixtureResult[dnfKey] === true;
+          fixturePartial[fixture] = partialResultLabel(
+            { ...fixtureResult },
+            packageManager,
+          );
 
           if (isDnf) {
             fixtureDnf[fixture] = true;
@@ -110,6 +117,7 @@ export const VariationTable = ({
           packageManager,
           fixtureValues,
           fixtureDnf,
+          fixturePartial,
         };
       }),
     [filteredPackageManagers, variationData],
@@ -168,6 +176,7 @@ export const VariationTable = ({
           header: () => <span className="font-bold">{getFixtureDisplayName(fixture)}</span>,
           cell: (info) => {
             const isDnf = info.row.original.fixtureDnf[fixture] === true;
+            const partial = info.row.original.fixturePartial[fixture];
             const value = info.getValue();
             if (isDnf) {
               return (
@@ -185,6 +194,11 @@ export const VariationTable = ({
                     {value.toFixed(decimals)}
                     {unit}
                   </span>
+                  {partial && (
+                    <div className="text-xs text-amber-700 dark:text-amber-400">
+                      {partial}
+                    </div>
+                  )}
                 </div>
               );
             }