diff --git a/.gitignore b/.gitignore index a09ba3fd..6f2a9356 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ pnpm-debug.log* # Generated/copied files public/data/**/* + +.tsbuildcache/node.tsbuildinfo +tsconfig.app.tsbuildinfo \ No newline at end of file diff --git a/scripts/codemod-v1-to-v2.test.ts b/scripts/codemod-v1-to-v2.test.ts new file mode 100644 index 00000000..0b03e2e4 --- /dev/null +++ b/scripts/codemod-v1-to-v2.test.ts @@ -0,0 +1,346 @@ +import { describe, it, expect } from "vitest"; +import { + splitExpertOverview, + widestScope, + upgradeV1ToV2, + PATHWAY_DESCRIPTION, + CORE_DRIVERS, + TRANSITION_ASSESSMENT, +} from "./codemod-v1-to-v2.ts"; +import type { PathwayMetadataV1 } from "../src/types/pathwayMetadata.v1.d.ts"; + +const V2_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +const WELL_FORMED = [ + "#### Pathway Description", + "", + "A description of the pathway.", + "", + "#### Core Drivers", + "", + "*Policy:* Policies drive it.", + "", + "#### Application to Transition Assessment", + "", + "How to apply it.", +].join("\n"); + +/** ACE-CNS-2024's shape: the Core Drivers heading lost its `####` markers. */ +const BARE_HEADING = [ + "#### Pathway Description", + "", + "A description of the pathway.", + "", + "Core Drivers", + "", + "*Policy:* Policies drive it.", + "", + "#### Application to Transition Assessment", + "", + "How to apply it.", +].join("\n"); + +describe("splitExpertOverview", () => { + it("splits the three ATX-headed sections", () => { + const s = splitExpertOverview(WELL_FORMED); + expect(s.get(PATHWAY_DESCRIPTION)).toBe("A description of the pathway."); + expect(s.get(CORE_DRIVERS)).toBe("*Policy:* Policies drive it."); + expect(s.get(TRANSITION_ASSESSMENT)).toBe("How to apply it."); + }); + + it("treats a bare section title as a heading (the ACE-CNS case)", () => { + const s = splitExpertOverview(BARE_HEADING); + // Without the tolerance, Core Drivers prose would land in the description + // and push it over the 2500-char limit. + expect(s.get(PATHWAY_DESCRIPTION)).toBe("A description of the pathway."); + expect(s.get(CORE_DRIVERS)).toBe("*Policy:* Policies drive it."); + }); + + it("does not treat a title mentioned mid-sentence as a heading", () => { + const s = splitExpertOverview( + [ + "#### Pathway Description", + "", + "The Core Drivers of this pathway are policy-led.", + ].join("\n"), + ); + expect(s.get(PATHWAY_DESCRIPTION)).toBe( + "The Core Drivers of this pathway are policy-led.", + ); + expect(s.has(CORE_DRIVERS)).toBe(false); + }); + + it("preserves markdown inside a section body", () => { + const s = splitExpertOverview( + ["#### Pathway Description", "", "para one.", "", "para two."].join("\n"), + ); + expect(s.get(PATHWAY_DESCRIPTION)).toBe("para one.\n\npara two."); + }); + + it("returns no sections for text with no recognised headings", () => { + expect(splitExpertOverview("Just prose.").size).toBe(0); + }); +}); + +function v1(over: Partial): PathwayMetadataV1 { + return { + $schema: "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + id: "X", + name: { full: "X" }, + description: "A pathway.", + publication: { + title: { full: "T" }, + publisher: { full: "TransitionZero" }, + year: 2024, + }, + pathwayType: "Normative", + geography: { global: true }, + sectors: [{ name: "Power", technologies: [] }], + expertOverview: WELL_FORMED, + metric: ["Capacity"], + keyFeatures: { + emissionsTrajectory: "Significant decrease", + energyEfficiency: "Moderate improvement", + energyDemand: "Minor increase", + electrification: "Significant increase", + policyTypes: ["Carbon price"], + technologyCostTrend: "Decrease", + emissionsScope: "CO2e (Kyoto)", + policyAmbition: "High ambition policies", + technologyCostsDetail: "Total costs", + newTechnologiesIncluded: ["CCUS"], + investmentNeeds: "By sector", + }, + ...over, + } as unknown as PathwayMetadataV1; +} + +describe("widestScope", () => { + it("uses the lone sector when there is only one", () => { + expect(widestScope(v1({})).sector).toBe("Power"); + }); + + it("uses across sectors for a multi-sector pathway", () => { + const doc = v1({ + sectors: [ + { name: "Power", technologies: [] }, + { name: "Steel", technologies: [] }, + ], + } as Partial); + expect(widestScope(doc).sector).toBe("across sectors"); + }); + + it("collapses a repeated sector name rather than calling it multi-sector", () => { + // pathwayMetadata has no uniqueItems on sectors, and the _full fixture + // deliberately lists Automotive twice. + const doc = v1({ + sectors: [ + { name: "Power", technologies: [] }, + { name: "Power", technologies: ["Solar"] }, + ], + } as Partial); + expect(widestScope(doc).sector).toBe("Power"); + }); + + it("prefers Global when the pathway is global, even with regions present", () => { + const doc = v1({ + geography: { global: true, regions: { "South East Asia": ["TH"] } }, + } as Partial); + expect(widestScope(doc).geography).toBe("Global"); + }); + + it("uses the lone region label (the ACE case)", () => { + const doc = v1({ + geography: { regions: { "South East Asia": ["TH", "VN"] } }, + } as Partial); + expect(widestScope(doc).geography).toBe("South East Asia"); + }); + + it("uses the lone country code", () => { + const doc = v1({ + geography: { country: ["TH"] }, + } as Partial); + expect(widestScope(doc).geography).toBe("TH"); + }); + + it("throws rather than guessing between several regions", () => { + const doc = v1({ + geography: { regions: { A: ["TH"], B: ["VN"] } }, + } as Partial); + expect(() => widestScope(doc)).toThrow(/ambiguous/); + }); + + it("throws rather than guessing between several countries", () => { + const doc = v1({ + geography: { country: ["TH", "VN"] }, + } as Partial); + expect(() => widestScope(doc)).toThrow(/ambiguous/); + }); + + it("throws when there is no geography at all", () => { + const doc = v1({ geography: {} } as Partial); + expect(() => widestScope(doc)).toThrow(/no geography/); + }); +}); + +describe("upgradeV1ToV2", () => { + it("repoints $schema at v2", () => { + expect(upgradeV1ToV2(v1({})).doc.$schema).toBe(V2_ID); + }); + + it("wraps every keyFeature as one entry at the widest scope", () => { + const { doc } = upgradeV1ToV2(v1({})); + const kf = doc.keyFeatures; + expect(Object.keys(kf)).toHaveLength(11); + for (const entries of Object.values(kf)) { + expect(entries).toHaveLength(1); + expect(entries[0].sector).toBe("Power"); + expect(entries[0].geography).toBe("Global"); + } + expect(kf.emissionsTrajectory[0].value).toBe("Significant decrease"); + }); + + it("nests an array-valued field rather than splatting it into entries", () => { + const doc = upgradeV1ToV2( + v1({ + keyFeatures: { + ...v1({}).keyFeatures, + policyTypes: ["Carbon price", "Subsidies"], + }, + } as Partial), + ).doc; + expect(doc.keyFeatures.policyTypes).toHaveLength(1); + expect(doc.keyFeatures.policyTypes[0].value).toEqual([ + "Carbon price", + "Subsidies", + ]); + }); + + it("moves the description section into pathwayDescription", () => { + const { doc } = upgradeV1ToV2(v1({})); + expect(doc.pathwayDescription).toBe("A description of the pathway."); + // The assessment section has no v2 field; it comes back in the result so the + // run report can print it rather than losing it. + expect("transitionAssessment" in doc).toBe(false); + }); + + it("discards pathwayOverview rather than folding it in", () => { + // #858 says to fold it in; the data owner confirmed on PR #898 that it should + // be dropped, because the two texts restate each other and the merged field + // reads as immediate self-repetition. + const { doc, droppedPathwayOverview } = upgradeV1ToV2( + v1({ pathwayOverview: "A short summary." } as Partial), + ); + expect(droppedPathwayOverview).toBe("A short summary.".length); + expect(doc.pathwayDescription).toBe("A description of the pathway."); + expect(doc.pathwayDescription).not.toContain("A short summary."); + }); + + it("reports nothing dropped when there was no pathwayOverview", () => { + expect(upgradeV1ToV2(v1({})).droppedPathwayOverview).toBe(0); + }); + + it("drops the v1 overview fields", () => { + const { doc } = upgradeV1ToV2( + v1({ pathwayOverview: "A short summary." } as Partial), + ); + expect("expertOverview" in doc).toBe(false); + expect("pathwayOverview" in doc).toBe(false); + }); + + it("scaffolds coreDrivers all-null and dependencies empty", () => { + const { doc } = upgradeV1ToV2(v1({})); + expect(Object.values(doc.coreDrivers)).toEqual([ + null, + null, + null, + null, + null, + null, + null, + ]); + expect(doc.dependencies).toEqual([]); + }); + + it("returns the retired Application-to-Transition-Assessment prose", () => { + expect(upgradeV1ToV2(v1({})).transitionAssessmentProse).toBe( + "How to apply it.", + ); + }); + + it("returns the Core Drivers prose it does not carry over", () => { + expect(upgradeV1ToV2(v1({})).coreDriversProse).toBe( + "*Policy:* Policies drive it.", + ); + }); + + it("nulls pathwayDescription when there is no description section", () => { + const { doc } = upgradeV1ToV2( + v1({ expertOverview: "No headings here." } as Partial), + ); + expect(doc.pathwayDescription).toBeNull(); + }); + + it("preserves unrelated fields and their order", () => { + const { doc } = upgradeV1ToV2( + v1({ modelYearNetzero: 2050 } as Partial), + ); + expect(doc.modelYearNetzero).toBe(2050); + // pathwayDescription takes expertOverview's slot; coreDrivers/dependencies + // follow keyFeatures. Key order keeps the data-file diffs readable. + const keys = Object.keys(doc); + expect(keys.indexOf("pathwayDescription")).toBeLessThan( + keys.indexOf("metric"), + ); + expect(keys.indexOf("coreDrivers")).toBeGreaterThan( + keys.indexOf("keyFeatures"), + ); + }); +}); + +describe("upgradeV1ToV2 — #461 technology reporting", () => { + it("reports nothing when every technology belongs to its sector", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Power", technologies: ["Solar", "Wind"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual([]); + }); + + it("reports a technology outside its sector's list", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Power", technologies: ["Solar", "Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual(["Power: Hydrogen Use"]); + }); + + it("reports technologies on a sector with no defined list", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Steel", technologies: ["Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual(["Steel: Hydrogen Use"]); + }); + + it("leaves the offending technologies in the output document", () => { + // Reporting, not fixing: deleting a technology someone recorded on purpose + // would be worse than a failing schema:check with a line in the report. + const { doc, unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Steel", technologies: ["Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toHaveLength(1); + expect(doc.sectors).toEqual([ + { name: "Steel", technologies: ["Hydrogen Use"] }, + ]); + }); + + it("is empty for the corpus shape — TransitionZero declares no technologies", () => { + expect(upgradeV1ToV2(v1({})).unscopedTechnologies).toEqual([]); + }); +}); diff --git a/scripts/codemod-v1-to-v2.ts b/scripts/codemod-v1-to-v2.ts new file mode 100644 index 00000000..575b0659 --- /dev/null +++ b/scripts/codemod-v1-to-v2.ts @@ -0,0 +1,393 @@ +/** + * One-shot codemod: pathwayMetadata v1 -> v2 (#858). + * + * Run over an explicit list of files, in place: + * + * npx ts-node --esm scripts/codemod-v1-to-v2.ts src/data/iea/IEA-NZE-2024.json ... + * npx ts-node --esm scripts/codemod-v1-to-v2.ts --dry-run src/data/iea + * + * A directory argument expands to the metadata files under it. Files already on v2 + * are skipped, so re-running is safe and the remaining 49 files are a re-run + * rather than a rewrite. + * + * This is a development tool, not a runtime path: the app does not convert v1 + * documents on load. Files that still carry the v1 $schema simply are not loaded + * once the loader points at v2. + * + * Two v1 prose fields are retired without replacement (confirmed with the data + * owner on PR #898): `pathwayOverview`, because it restates the description + * almost verbatim; and the '#### Application to Transition Assessment' section, + * because the new UI does not display it. Neither has a v2 home, so the section + * text is printed in the run report alongside the Core Drivers prose rather than + * being dropped silently. + * + * It also reports, without touching, technologies attached to a sector that does + * not list them (#461). Auto-fixing would mean either deleting a technology + * someone deliberately recorded or inventing a sector's taxonomy -- both worse + * than a line in the report, and `npm run schema:check` refuses the file anyway. + * + * What it does NOT do: populate `coreDrivers`. The v1 "#### Core Drivers" prose + * cannot be mapped onto the 7 named fields mechanically -- several paragraphs + * exceed the 500-char cap, the italic labels do not correspond 1:1 to the field + * names, and each section has unlabeled paragraphs with no destination. Per #858 + * the fields are scaffolded null for hand-authoring; the prose is printed in the + * report so it is not lost track of. + */ +import { promises as fs } from "node:fs"; +import { join, extname } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { PathwayMetadataV1 } from "../src/types/pathwayMetadata.v1.d.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +import { technologyBelongsToSector } from "../src/utils/timeseriesTaxonomy.ts"; + +const V1_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v1.json"; +const V2_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +/** Widest sentinels, mirroring src/utils/validateScopes.ts. */ +const ACROSS_SECTORS = "across sectors"; +const GLOBAL_SCOPE = "Global"; + +/** + * The three sections every v1 `expertOverview` is built from. Verified against all + * 56 files: 55 have all three, and ACE-CNS-2024 is missing "Core Drivers" only + * because its heading lost its `####` markers (see splitExpertOverview). + */ +export const PATHWAY_DESCRIPTION = "Pathway Description"; +export const CORE_DRIVERS = "Core Drivers"; +export const TRANSITION_ASSESSMENT = "Application to Transition Assessment"; +export const SECTION_TITLES = [ + PATHWAY_DESCRIPTION, + CORE_DRIVERS, + TRANSITION_ASSESSMENT, +] as const; + +const CORE_DRIVER_FIELDS = [ + "policies", + "emissionsTargets", + "technologyCosts", + "investmentChange", + "macroeconomicDrivers", + "behavioralShifts", + "otherDrivers", +] as const; + +/** + * Split a v1 `expertOverview` into its named sections. + * + * Headings are ATX (`#### Core Drivers`), but a bare line whose entire content is + * a known section title also counts. That tolerance exists for exactly one file: + * ACE-CNS-2024.json lost the `####` on its "Core Drivers" heading, which would + * otherwise fold 1.5 KB of core-drivers prose into pathwayDescription and push it + * past the 2500-char limit. Treating the bare title as a heading is safe because + * the strings are long and specific enough not to occur as body text. + */ +export function splitExpertOverview(text: string): Map { + const sections = new Map(); + const lines = text.split("\n"); + let current: string | null = null; + let buffer: string[] = []; + + const flush = () => { + if (current !== null) sections.set(current, buffer.join("\n").trim()); + buffer = []; + }; + + for (const line of lines) { + const stripped = line.trim(); + const atx = /^#{1,6}\s*(.+?)\s*$/.exec(stripped); + const heading = atx ? atx[1] : stripped; + const known = SECTION_TITLES.find((t) => t === heading); + if (known && (atx || stripped === known)) { + flush(); + current = known; + continue; + } + if (current !== null) buffer.push(line); + } + flush(); + return sections; +} + +/** + * The widest scope an entry on this pathway can carry, per #858: `across sectors` + * for a multi-sector pathway else its lone sector, and `Global` else the + * pathway's single declared region or country. + * + * Throws rather than guessing when a pathway declares several regions or several + * standalone countries without `global`, since which of them is "widest" is an + * authoring decision (`across regions` exists for that case). No file in the corpus + * hits this today -- all 56 resolve. + */ +export function widestScope(doc: PathwayMetadataV1): { + sector: string; + geography: string; +} { + const names = [...new Set((doc.sectors ?? []).map((s) => s.name))]; + if (names.length === 0) throw new Error("pathway declares no sectors"); + const sector = names.length > 1 ? ACROSS_SECTORS : names[0]; + + const geo = doc.geography ?? {}; + const regions = Object.keys(geo.regions ?? {}); + const countries = geo.country ?? []; + + let geography: string; + if (geo.global === true) { + geography = GLOBAL_SCOPE; + } else if (regions.length === 1) { + geography = regions[0]; + } else if (regions.length > 1) { + throw new Error( + `${regions.length} regions and no "global" flag: widest geography is ambiguous ` + + `(consider "across regions"): ${regions.join(", ")}`, + ); + } else if (countries.length === 1) { + geography = countries[0]; + } else if (countries.length > 1) { + throw new Error( + `${countries.length} countries and no "global" flag or region: widest geography is ambiguous`, + ); + } else { + throw new Error("pathway declares no geography"); + } + + return { sector, geography }; +} + +/** Wrap one v1 value as a single scoped entry at the given scope. */ +function scoped(sector: string, geography: string, value: T) { + return [{ sector, geography, value }]; +} + +export interface UpgradeResult { + doc: PathwayMetadataV2; + /** Prose with no destination in v2 -- reported so it is not lost track of. */ + coreDriversProse: string; + /** Ditto: the new UI does not show this, so v2 has no field for it. */ + transitionAssessmentProse: string; + scope: { sector: string; geography: string }; + /** Chars of `pathwayOverview` discarded, for the run report. */ + droppedPathwayOverview: number; + /** + * `"Sector: Technology"` for each technology its sector does not list (#461). + * Carried straight over from v1, so this reports pre-existing data rather than + * anything the codemod introduced. + */ + unscopedTechnologies: string[]; +} + +/** Transform one v1 document into its v2 equivalent. Pure; does no I/O. */ +export function upgradeV1ToV2(doc: PathwayMetadataV1): UpgradeResult { + const scope = widestScope(doc); + const { sector, geography } = scope; + const kf = doc.keyFeatures; + + const sections = splitExpertOverview(doc.expertOverview ?? ""); + const description = sections.get(PATHWAY_DESCRIPTION) ?? ""; + const assessment = sections.get(TRANSITION_ASSESSMENT) ?? ""; + const coreDriversProse = sections.get(CORE_DRIVERS) ?? ""; + + // `pathwayOverview` is DISCARDED, not folded into pathwayDescription. + // + // #858 says to move it in, and an earlier version of this script did. Reading + // the output showed why that is wrong: the two texts restate each other almost + // verbatim, so the merged field opens by saying the same thing twice. IEA-STEPS + // for instance had "STEPS provides a sense of the energy sector's direction of + // travel today, based on the latest market data, technology costs and in-depth + // analysis of the prevailing policy settings..." immediately followed by "STEPS + // projects the energy sector's current direction of travel, based on the latest + // market data, technology costs and in-depth analysis of the stated policies...". + // + // Confirmed with the data owner (Jacob, PR #898): the field was already out of + // use and should be removed without replacement. It has no readers in the app, + // so nothing observable is lost. + const droppedPathwayOverview = doc.pathwayOverview?.trim().length ?? 0; + + const out: Record = {}; + for (const [key, value] of Object.entries(doc)) { + switch (key) { + case "$schema": + out.$schema = V2_ID; + break; + case "pathwayOverview": + // Superseded; emitted below in expertOverview's position. + break; + case "expertOverview": + out.pathwayDescription = description.length > 0 ? description : null; + break; + case "keyFeatures": + out.keyFeatures = { + emissionsTrajectory: scoped( + sector, + geography, + kf.emissionsTrajectory, + ), + energyEfficiency: scoped(sector, geography, kf.energyEfficiency), + energyDemand: scoped(sector, geography, kf.energyDemand), + electrification: scoped(sector, geography, kf.electrification), + policyTypes: scoped(sector, geography, kf.policyTypes), + technologyCostTrend: scoped( + sector, + geography, + kf.technologyCostTrend, + ), + emissionsScope: scoped(sector, geography, kf.emissionsScope), + policyAmbition: scoped(sector, geography, kf.policyAmbition), + technologyCostsDetail: scoped( + sector, + geography, + kf.technologyCostsDetail, + ), + newTechnologiesIncluded: scoped( + sector, + geography, + kf.newTechnologiesIncluded, + ), + investmentNeeds: scoped(sector, geography, kf.investmentNeeds), + }; + // Scaffolded for hand-authoring; see the file header. + out.coreDrivers = Object.fromEntries( + CORE_DRIVER_FIELDS.map((f) => [f, null]), + ); + out.dependencies = []; + break; + default: + out[key] = value; + } + } + + const unscopedTechnologies = (doc.sectors ?? []).flatMap((s) => + (s.technologies ?? []) + .filter((t) => technologyBelongsToSector(t, s.name) !== "yes") + .map((t) => `${s.name}: ${t}`), + ); + + return { + doc: out as unknown as PathwayMetadataV2, + unscopedTechnologies, + coreDriversProse, + transitionAssessmentProse: assessment, + scope, + droppedPathwayOverview, + }; +} + +async function metadataFilesUnder(path: string): Promise { + const stat = await fs.stat(path); + if (!stat.isDirectory()) return [path]; + const dirents = await fs.readdir(path, { withFileTypes: true }); + const found: string[] = []; + for (const d of dirents) { + const full = join(path, d.name); + if (d.isDirectory()) found.push(...(await metadataFilesUnder(full))); + else if (d.isFile() && extname(d.name) === ".json") found.push(full); + } + return found.sort(); +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const paths = args.filter((a) => !a.startsWith("--")); + + if (paths.length === 0) { + console.error( + "usage: codemod-v1-to-v2.ts [--dry-run] [...]\n" + + "Refusing to run with no explicit target.", + ); + process.exit(1); + } + + const files = (await Promise.all(paths.map(metadataFilesUnder))).flat(); + const report: string[] = []; + let migrated = 0; + let skipped = 0; + + for (const file of files) { + const raw = await fs.readFile(file, "utf8"); + const parsed = JSON.parse(raw) as { $schema?: string }; + if (parsed.$schema === V2_ID) { + skipped++; + continue; + } + if (parsed.$schema !== V1_ID) { + // Timeseries files and anything else non-metadata. + skipped++; + continue; + } + + const doc = parsed as unknown as PathwayMetadataV1; + let result: UpgradeResult; + try { + result = upgradeV1ToV2(doc); + } catch (e) { + console.error(`✖ ${file}: ${String(e instanceof Error ? e.message : e)}`); + process.exitCode = 1; + continue; + } + + const { scope, coreDriversProse, droppedPathwayOverview } = result; + const descLength = result.doc.pathwayDescription?.length ?? 0; + console.info( + `✔ ${file}\n` + + ` scope: (${scope.sector}, ${scope.geography})` + + ` pathwayDescription: ${descLength} chars` + + (droppedPathwayOverview > 0 + ? ` [dropped ${droppedPathwayOverview}-char pathwayOverview]` + : ""), + ); + if (result.unscopedTechnologies.length > 0) { + // stderr, because this needs a human before the file will pass + // schema:check -- but not a non-zero exit, since the migration itself + // succeeded and the offending data predates it. + console.error( + ` #461: technologies outside their sector's list, left as-is: ` + + result.unscopedTechnologies.join("; "), + ); + } + + const orphaned: string[] = []; + if (coreDriversProse.length > 0) { + orphaned.push(`### ${CORE_DRIVERS}\n\n${coreDriversProse}`); + } else { + console.info(` note: no "${CORE_DRIVERS}" section found`); + } + if (result.transitionAssessmentProse.length > 0) { + orphaned.push( + `### ${TRANSITION_ASSESSMENT}\n\n${result.transitionAssessmentProse}`, + ); + } + if (orphaned.length > 0) { + report.push(`## ${file}\n\n${orphaned.join("\n\n")}\n`); + } + + if (!dryRun) { + await fs.writeFile(file, `${JSON.stringify(result.doc, null, 2)}\n`); + } + migrated++; + } + + console.info( + `\n${dryRun ? "Would migrate" : "Migrated"} ${migrated} file(s); skipped ${skipped}.`, + ); + + if (report.length > 0) { + console.info( + "\n--- v1 prose with no v2 destination ---\n" + + `"${CORE_DRIVERS}" becomes the structured coreDrivers object, scaffolded null here;\n` + + `"${TRANSITION_ASSESSMENT}" is retired because the new UI does not display it.\n` + + "Both are reproduced below so the text is not lost.\n", + ); + console.info(report.join("\n")); + } + if (!dryRun) { + console.info("Run `npm run schema:check` and `npx prettier --write` next."); + } +} + +// Only run when invoked directly, so the exported helpers stay importable by tests. +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch((e: unknown) => { + console.error(String(e instanceof Error ? e.stack : e)); + process.exit(1); + }); +} diff --git a/scripts/schema-check-files.ts b/scripts/schema-check-files.ts index d079d859..76c38d55 100644 --- a/scripts/schema-check-files.ts +++ b/scripts/schema-check-files.ts @@ -5,8 +5,14 @@ import type { FileEntry } from "../src/utils/validateData.ts"; import { validateFilesBySchema } from "../src/utils/validateData.ts"; import { decideIncludeInvalid } from "../src/utils/loadData.ts"; import pathwayMetadata from "../src/schema/pathwayMetadata.v1.json" with { type: "json" }; +import pathwayMetadataV2 from "../src/schema/pathwayMetadata.v2.json" with { type: "json" }; import pathwayTimeseries from "../src/schema/pathwayTimeseries.v1.json" with { type: "json" }; import { commonSchemas } from "../src/schema/common/index.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +import { + PATHWAY_METADATA_V2_ID, + validateScopedEntries, +} from "../src/utils/validateScopes.ts"; async function run(dir: string) { async function getJsonFilesRecursive(base: string): Promise { @@ -34,10 +40,29 @@ async function run(dir: string) { const { valid, invalid } = validateFilesBySchema(entries, [ pathwayMetadata, + pathwayMetadataV2, pathwayTimeseries, ...commonSchemas, ]); - return { dir, validCount: valid.length, invalid }; + + // Second pass over the AJV-valid v2 documents for the cross-field scope + // constraint draft-07 cannot express — see src/utils/validateScopes.ts. A + // document that fails here is reported exactly like a schema failure, so a + // mistyped region label breaks the build instead of silently matching nothing. + const scopeProblems = valid + .filter((r) => r.schemaId === PATHWAY_METADATA_V2_ID) + .map((r) => ({ + name: r.name, + errors: validateScopedEntries(r.data as PathwayMetadataV2), + })) + .filter((p) => p.errors.length > 0); + + const badNames = new Set(scopeProblems.map((p) => p.name)); + return { + dir, + validCount: valid.filter((r) => !badNames.has(r.name)).length, + invalid: [...invalid, ...scopeProblems], + }; } async function main() { @@ -73,8 +98,12 @@ async function main() { filesWithIssues += r.invalid.length; for (const p of r.invalid) { - // Emit up to N errors per file as GH annotations - const file = join(r.dir, p.name); + // Emit up to N errors per file as GH annotations. + // `p.name` already carries the directory — getJsonFilesRecursive builds it + // with join(base, d.name) — so re-prepending r.dir produced paths like + // "src/data/src/data/foo.json" and the annotations pointed nowhere. Latent + // until now because it only shows when a file actually fails. + const file = p.name; const errs = p.errors.slice(0, 50); for (const e of errs) { console.log(`::${annotate} file=${file}::${e}`); diff --git a/src/components/KeyFeatures.test.tsx b/src/components/KeyFeatures.test.tsx index d452faf1..9fc1b61b 100644 --- a/src/components/KeyFeatures.test.tsx +++ b/src/components/KeyFeatures.test.tsx @@ -3,19 +3,28 @@ import { render, screen } from "@testing-library/react"; import KeyFeatures from "./KeyFeatures"; import type { PathwayMetadataType } from "../types"; +/** + * v2 scopes every keyFeature as {sector, geography, value} entries (#858). These + * fixtures use a single widest-scope entry per field — what the codemod produces — + * so the rendering assertions below still describe v1's output. + */ +const wide = (value: T) => [ + { sector: "across sectors", geography: "Global", value }, +]; + const mockKeyFeatures: PathwayMetadataType["keyFeatures"] = { - emissionsScope: "CO2", - emissionsTrajectory: "Moderate decrease", - energyEfficiency: "Minor improvement", - energyDemand: "Low or no change", - electrification: "Moderate increase", - policyTypes: ["Carbon price", "Subsidies"], - policyAmbition: "NDCs incl. conditional targets", - newTechnologiesIncluded: ["CCUS", "Battery storage"], - technologyCostTrend: "Decrease", - technologyCostsDetail: "Total costs", - investmentNeeds: "By technology", -}; + emissionsScope: wide("CO2"), + emissionsTrajectory: wide("Moderate decrease"), + energyEfficiency: wide("Minor improvement"), + energyDemand: wide("Low or no change"), + electrification: wide("Moderate increase"), + policyTypes: wide(["Carbon price", "Subsidies"]), + policyAmbition: wide("NDCs incl. conditional targets"), + newTechnologiesIncluded: wide(["CCUS", "Battery storage"]), + technologyCostTrend: wide("Decrease"), + technologyCostsDetail: wide("Total costs"), + investmentNeeds: wide("By technology"), +} as unknown as PathwayMetadataType["keyFeatures"]; describe("KeyFeatures", () => { it("renders all four group headers", () => { @@ -86,7 +95,7 @@ describe("KeyFeatures", () => { it("sentiment feature: unfavorable value uses red scale color", () => { const unfavorable = { ...mockKeyFeatures, - emissionsTrajectory: "Significant increase", + emissionsTrajectory: wide("Significant increase"), } as unknown as PathwayMetadataType["keyFeatures"]; render(); @@ -101,7 +110,9 @@ describe("KeyFeatures", () => { it("sentiment feature: no-info value renders a Badge, not a colored text span", () => { const noInfo = { ...mockKeyFeatures, - emissionsTrajectory: undefined, + // v2's way of saying "nothing authored at any scope" is an empty entry + // array, not a missing field — the field stays required. + emissionsTrajectory: [], } as unknown as PathwayMetadataType["keyFeatures"]; render(); diff --git a/src/components/KeyFeatures.tsx b/src/components/KeyFeatures.tsx index d68ce2a9..ab65907b 100644 --- a/src/components/KeyFeatures.tsx +++ b/src/components/KeyFeatures.tsx @@ -1,6 +1,7 @@ import React from "react"; import { PathwayMetadataType } from "../types"; import { getKeyFeatureTooltip } from "../utils/tooltipUtils"; +import { widestValue } from "../utils/keyFeatureScope"; import TextWithTooltip from "./TextWithTooltip"; import Badge from "./Badge"; import SentimentScale, { getSentimentPalette } from "./SentimentScale"; @@ -238,7 +239,11 @@ export const FeatureItem: React.FC = ({ labelClassName = "text-xs font-medium text-rmigray-500", showLabel = true, }) => { - const rawValue = keyFeatures[feature.key]; + // v2 stores each feature as scoped {sector, geography, value} entries (#858). + // Render the value at the broadest scope, which reproduces v1's output exactly + // for codemod-migrated data (one entry, at its widest scope). #869 replaces this + // with a scope-aware resolver and #859 adds the badge that names the scope. + const rawValue = widestValue(keyFeatures[feature.key]); const label =

{feature.label}

; diff --git a/src/data/README.md b/src/data/README.md index 91bac4ac..e0021f1c 100644 --- a/src/data/README.md +++ b/src/data/README.md @@ -1,110 +1,275 @@ # Pathway metadata for the tpr repo The `src/data` directory in the [tpr](https://github.com/RMI/tpr) repo contains all of the data shown on the Transition Pathways Repository site. -Each JSON file in this directory contains one or more pathway definitions. +Each JSON file in this directory contains one pathway definition, alongside optional `*_timeseries.json` files holding that pathway's data series. ## Schema and Validation -The JSON files have a strict, specific format that needs to be followed, which is defined by the JSON schema file in this repo at [pbtar_schema.json](https://github.com/RMI/tpr/blob/main/src/schema/schema.json). +The JSON files have a strict, specific format, defined by the JSON schema files in [`src/schema/`](https://github.com/RMI/tpr/tree/main/src/schema). +The schema is split across several files: the pathway metadata schema itself, plus shared definitions under `src/schema/common/` that it references (country codes, sector and technology names, publication details, and so on). -The schema file defines a number of mandatory fields which must be included. -Additionally, the structure, the data types, and in some cases the allowed values for a given key must be correct for things to work as expected. -This repo has CI/CD setup to validate any new JSON added in a PR against the schema. -Therefore, any new JSON added through a PR on main must pass all of the tests before being merged. +The schema defines a number of mandatory fields which must be included. +The structure, the data types, and in many cases the allowed values for a given key must all be correct for things to work as expected. +This repo has CI/CD set up to validate any new JSON added in a PR, so any new JSON added through a PR on `main` must pass before being merged. -After preparing a JSON file, you can run `npm run json:check`, which will trigger validation (locally) against all JSON files in this directory. -You can preview a JSON file as it will appear in the UI with `npm run dev`. -To add a new Pathway, a new JSON file in the correct format needs to be added to this repo in a pull request on `main`. +After preparing a JSON file, validate it locally with: -## Examples +```bash +npm run schema:check +``` -To see example files, look in this directory, or `testdata/valid/`. +That checks every file under `src/data` and `testdata/valid`. You can preview a file as it will appear in the UI with `npm run dev`. -## Creating new pathway files +If you have changed a schema rather than a data file, run `npm run schema` instead — it validates and then regenerates the TypeScript types in `src/types/`. Those are checked in, and CI fails if they are out of date. -To facilitate creating a new JSON file in the appropriate format using R, we have created the following two functions to validate a `` object against the schema in this repo, and to write a valid nested `` to a JSON file. -These functions can be copy-pasted to your R console and then they're available to use on any `` you have in your environment. -These functions require the following R packages to be installed in your environment: `jsonvalidate`, `jsonlite`, `dplyr`, `tidyr`, `stringr`, and `purrr`. +### Two schema versions -```r -# if schema_url is not provided, it will validate against the current PROD schema. +Two versions of the metadata schema currently exist: + +- `pathwayMetadata.v2.json` — the current format. **New pathways should use this.** +- `pathwayMetadata.v1.json` — the previous format, still present so existing files remain valid. + +Every file declares which one it follows via its own `$schema` key, and the validator routes each file to the matching schema. A file on either version will pass `npm run schema:check`. + +**Only v2 files are loaded by the site.** A file still on v1 validates but does not appear in the app; the dev server logs how many were skipped. Migration of the remaining v1 files is in progress. + +## The v2 format + +A complete, valid example lives at [`testdata/valid/pathwayMetadata_v2_full.json`](../../testdata/valid/pathwayMetadata_v2_full.json), and a minimal one at `pathwayMetadata_v2_minimal.json`. Real pathways are in the publisher subdirectories here. + +If you are used to the v1 format, these are the differences that matter: + +### keyFeatures are scoped + +In v1 each of the 11 key features held a single value for the whole pathway. In v2 each holds an **array of entries**, so a pathway can record different values for different parts of its coverage: + +```json +"keyFeatures": { + "emissionsTrajectory": [ + { "sector": "across sectors", "geography": "Global", "value": "Moderate decrease" }, + { "sector": "Power", "geography": "South East Asia", "value": "Significant decrease" } + ] +} +``` + +- `sector` is one of the sector names, or `"across sectors"` meaning "all of the sectors this pathway covers". +- `geography` is `"Global"`, one of the region labels used in this pathway's own `geography.regions`, or one of its country codes. +- `value` is exactly what v1 held for that field — the same allowed values. For the two fields that were arrays in v1 (`policyTypes`, `newTechnologiesIncluded`), `value` is still an array. + +Both `sector` and `geography` must be something the pathway actually declares, or one of the widest sentinels. A region label that does not appear in the pathway's own `geography.regions` is rejected, which is what catches a typo like `"Southeast Asia"` where the pathway says `"South East Asia"`. + +If a feature does not vary, give it **one entry at the widest scope that applies** — `across sectors` for a multi-sector pathway (otherwise its only sector), and `Global` for a global pathway (otherwise its region or country). -validate_json <- function(json_obj, schema_url = NULL) { - if (is.null(schema_url)) { - schema_url <- "https://raw.githubusercontent.com/RMI/tpr/refs/heads/main/pbtar_schema.json" +An empty array means nothing is recorded at any scope. That is different from an entry whose `value` is `"No information"`, which is a deliberate statement that this scope has no data. + +### sectors carry their segments, and the scope sentinels changed spelling + +`sectors[]` entries gain an optional `segments` list alongside `technologies` — which +parts of that sector's value chain the pathway covers: + +```json +"sectors": [ + { + "name": "Power", + "technologies": ["Solar", "Wind", "Coal"], + "segments": ["Power generation", "Energy storage"] } - json_schema <- readr::read_file(file = schema_url) - validation <- - jsonvalidate::json_validate( - json = jsonlite::toJSON(json_obj, auto_unbox = TRUE), - schema = json_schema, - verbose = TRUE, - greedy = TRUE, - engine = "ajv" - ) - if (!validation) { - errors <- - attr(validation, "errors") |> - dplyr::mutate(key = stringr::str_extract(instancePath, "[a-z]+")) |> - tidyr::unnest(params) |> - dplyr::rename(input = data) |> - dplyr::select(dplyr::any_of(c("input", "key", "message", "allowedValues"))) - return(errors) +] +``` + +Optional on purpose: only Power, Steel and Aviation have segments defined, and leaving +it out means "not recorded" where `[]` would claim the pathway covers none of them. + +The two widest-scope sentinels are now spelled as the cookbook spells them: +**`across sectors`** (was `cross-sector`) and **`across regions`** (was `cross-region`, +a value the cookbook never defined). + +### expertOverview is replaced by pathwayDescription + +v1's single `expertOverview` was one markdown document containing three sections. Only +one survives as prose in v2: + +- `pathwayDescription` (required, may be `null`, max 2500 chars) — the narrative description. +- the "Core Drivers" section becomes the structured `coreDrivers` object below. +- the "Application to Transition Assessment" section is **retired** — the new UI + does not display it, so v2 has no field for it. + +v1's separate `pathwayOverview` field is **retired without replacement**. Do not +carry it into `pathwayDescription`: in practice the two texts restate each other, +so merging them reads as immediate self-repetition. The field had no readers in +the app, so nothing is lost by dropping it. + +### coreDrivers and dependencies are new + +`coreDrivers` is an object with seven fields, all required but each allowed to be `null`. `null` means "not a core driver for this pathway", which is deliberately different from a driver that applies but has not been described. + +```json +"coreDrivers": { + "policies": "Carbon pricing across both covered sectors.", + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null +} +``` + +`dependencies` is an array — use `[]` if there are none. Each entry names a category, describes the dependency, scopes it to one of the pathway's sectors, and states how strong the evidence is: + +```json +"dependencies": [ + { + "dependency_name": "Infrastructure and logistics", + "dependency_description": "Grid buildout must keep pace with renewable additions.", + "sector": "Power", + "evidence_type": "Quantitative" } +] +``` + +The allowed values for `dependency_name` and `evidence_type` are listed in the schema. + +### dataAvailability describes where each metric's data lives + +`dataAvailability` is **optional** — a pathway without it is still valid, and authoring is incremental. Once present it has two halves: `overall`, a prose summary (or `null`), and `byMetric`, one row per (metric, sector, sector segment, geography set). + +```json +"dataAvailability": { + "overall": "The full timeseries file covers Power at annual resolution from 2022 to 2050.", + "byMetric": [ + { + "metricName": "Capacity", + "sector": "Power", + "sectorSegment": ["Fuel extraction and processing", "Power generation"], + "geography": ["Global", "South East Asia", "SG"], + "timeResolution": "1-year steps", + "dataFormat": "Tabular", + "granularity": ["Solar", "Wind", "Coal"], + "scopeLimitations": "Excludes off-grid generation." + } + ] } +``` + +Four things to know before authoring: + +**Write a row for every allowable (sector, metric) pair**, not only the ones with data. An uncovered pair is recorded as `"Not covered"`, not omitted — an omitted row reads as "nobody has looked at this yet". + +**`Unspecified` and `Not covered` are different, and neither is `null`.** `Unspecified` means the pathway covers this pair but does not state the value. `Not covered` means it does not cover the pair at all — so it applies to the _whole row_, and `npm run schema:check` rejects a row that mixes it with authored values. This is the same distinction `keyFeatures` draws with its explicit `"No information"`. + +**`geography`, `sectorSegment` and `granularity` are all lists.** Every geography member must be a region label or country the pathway declares in its own `geography`, and every segment must be one of its sector's — or one of the two sentinels, which must then be the list's only member. One metric can be projected at several levels at once, and cover more than one segment. + +**`dataFormat` describes the source publication only** — `Tabular`, `Text`, `Figure` or `Not covered`. Whether this repo hosts a copy is not a judgement about the pathway; the app derives that from the timeseries index. Where values appear in more than one form, the most extractable wins: `Tabular` before `Text` before `Figure`. + +**`metricName` is a different list from the pathway's own `metric` field.** The pathway-level `metric` is the five Power metrics that drive the search filter; the availability row key adds the ones a pathway reports but the tool does not plot — transmission lines, technology cost, investment requirement, asset lifetime — plus Steel's and Aviation's own metrics. Two cookbook variables, deliberately different. So a row may name a metric that is not in the pathway's `metric` list, which is exactly how an uncovered pair gets its `Not covered` row. + +Allowed values live in `src/schema/common/dataAvailability.v1.json` (time resolution, data format, the non-technology granularity members), `dataAvailabilityMetric.v1.json` (the row key) and `sectorSegment.v1.json`, and follow cookbook `tpr_cookbook_20260929`. + +## Migrating an existing v1 file + +There is a script for this — don't do it by hand: + +```bash +npx ts-node --esm scripts/codemod-v1-to-v2.ts --dry-run src/data/ +``` + +Drop `--dry-run` to write the changes. It rewrites files in place, skips anything already on v2, and prints the scope it chose for each file. + +It does **not** fill in `coreDrivers` — the v1 "Core Drivers" prose does not map onto the seven named fields mechanically, so the script scaffolds them all to `null` and prints the original text for whoever authors them. Run `npx prettier --write` on the files afterwards, then `npm run schema:check`. + +## Creating new pathway files +To create a new file in the appropriate format using R, the function below writes a nested `` out as JSON. +It can be copy-pasted to your R console and requires the `jsonlite` package. + +```r write_json <- function(json_obj, file) { - validation <- validate_json(json_obj) - if (!is.data.frame(validation)) { - jsonlite::write_json( - x = json_obj, - path = file, - auto_unbox = TRUE, - pretty = TRUE - ) - return(invisible()) - } - validation + jsonlite::write_json( + x = json_obj, + path = file, + auto_unbox = TRUE, + pretty = TRUE, + null = "null" + ) } ``` -Once the above functions have been loaded in your R environment, a new `` can be created, and then validated and exported as a JSON file using these functions like so... +Note on validating from R: the schema is split across several files that reference each other by URL, and the usual `jsonvalidate::json_validate()` call takes a single schema and will not fetch those references. Write the file first, then validate it with `npm run schema:check`, which resolves them correctly and also runs the checks that JSON Schema alone cannot express — such as confirming each `sector` and `geography` is one the pathway declares. + +Once the function above is loaded, a new pathway can be created and written out like so. ```r -# Note that single-element arrays must be wrapped with I(), the identity function, to ensure that `jsonlite` processes them as arrays, rather than length-1 vectors (everything is a vector in R). +# Single-element vectors must be wrapped with I(), the identity function, so that +# `jsonlite` writes them as arrays rather than as bare values (everything is a +# vector in R). Fields that must be `null` rather than absent use NA... see below. + +scoped <- function(value) { + list(list(sector = "Power", geography = "VN", value = value)) +} new_pathway_metadata <- list( - list( - id = "R-import-example", - name = "R Import Pathway", - description = "Pathway Imported from R", - pathwayType = "Normative", - modelYearEnd = 2050, - modelTempIncrease = 1.5, - geography = list("Global", "US", "Europe"), - sectors = list( - list(name = "Power", technologies = c("Coal", "Wind")), - list(name = "Steel", technologies = I(c("Other"))) - ), - publisher = "Example Publisher", - publicationYear = 2021, - metric = I(c("Capacity")), # `I()` is necessary so that jsonlite parses it as a length 1 array - expertOverview = "Text based expert recommendation long.", - dataSource = list( - description = "Data source description.", - url = "https://www.example.com/", - downloadAvailable = FALSE + `$schema` = "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", + id = "R-import-example", + name = list(full = "R Import Pathway", short = "R Example"), + description = "Pathway imported from R.", # must end in a period + publication = list( + title = list(full = "Example Publication"), + publisher = list(full = "TransitionZero"), + year = 2021, + links = list( + list(description = "Report", url = "https://www.example.com/") + ) + ), + pathwayType = "Normative", + modelYearEnd = 2050, + modelTempIncrease = 1.5, + geography = list( + regions = list(`South East Asia` = I(c("VN", "TH"))), + country = I(c("VN")) + ), + sectors = list( + list(name = "Power", technologies = I(c("Coal", "Wind"))) + ), + pathwayDescription = "A short narrative description of the pathway.", + metric = I(c("Capacity")), + keyFeatures = list( + emissionsTrajectory = scoped("Moderate decrease"), + energyEfficiency = scoped("Moderate improvement"), + energyDemand = scoped("Minor increase"), + electrification = scoped("Significant increase"), + policyTypes = scoped(I(c("Carbon price"))), + technologyCostTrend = scoped("Decrease"), + emissionsScope = scoped("CO2"), + policyAmbition = scoped("Current/legislated policies"), + technologyCostsDetail = scoped("Total costs"), + newTechnologiesIncluded = scoped(I(c("Battery storage"))), + investmentNeeds = scoped("By sector") + ), + coreDrivers = list( + policies = NULL, emissionsTargets = NULL, technologyCosts = NULL, + investmentChange = NULL, macroeconomicDrivers = NULL, + behavioralShifts = NULL, otherDrivers = NULL + ), + dependencies = list( + list( + dependency_name = "Technology", + dependency_description = "Grid capacity must expand.", + sector = "Power", + evidence_type = "Qualitative" ) ) ) -validate_json(new_pathway_metadata) - -write_json(new_pathway_metadata, "test.json") +write_json(new_pathway_metadata, "src/data/example-publisher/EXAMPLE-2021.json") ``` -If the `` is not valid, the functions will return a data frame with information about what was invalid. +One R-specific note: `jsonlite` writes an R `NULL` as `{}` by default, which the schema rejects. That is why `write_json` above passes `null = "null"` — with it, the seven `NULL` entries in `coreDrivers` come out as JSON `null` as intended. All seven keys are required even when every one of them is null, so keep them all. + +The example above has been run as written, and the file it produces passes `npm run schema:check`. + +## Keeping this up to date This README should be the definitive source of information about these JSON files and how to add them or modify them. As this repo is currently under heavy development, such details may change rapidly, and this README should be kept up to date with those changes as they happen. diff --git a/src/data/asean-centre-for-energy/ACE-ATS-2024.json b/src/data/asean-centre-for-energy/ACE-ATS-2024.json index ce43752f..5eebeae7 100644 --- a/src/data/asean-centre-for-energy/ACE-ATS-2024.json +++ b/src/data/asean-centre-for-energy/ACE-ATS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "ACE-ATS-2024", "publication": { "title": { @@ -72,7 +72,7 @@ "technologies": [] } ], - "expertOverview": "#### Pathway Description\n\nThe ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE ASEAN Member States Targets Scenario (ATS) is an exploratory pathway which assumes that every ASEAN member state fully achieves its unconditional transition targets on time, including unconditional goals set via Nationally Determined Contributions (NDCs). This pathway models the impact of existing policies, such as national power development plans (PDPs), and the implementation of these stated targets. The ATS pathway models more ambitious energy efficiency targets, as well as a faster deployment of low-carbon energy technologies, especially from renewable energy. The ATS pathway does not include more aspirational national objectives, such as conditional NDCs or long-term net zero goals. For examples of pathways that include these policies, see the ACE Regional Aspiration Scenario (RAS) and IEA Announced Pledges Scenario (APS).\n\nPower generation emissions decline -2.4% per year, resulting in a total decrease of 74% relative to power emissions in the Baseline Scenario. However, the ATS pathway projects to an average annual emissions increase of 0.8% across the full energy sector. \n\n#### Core Drivers\n\nThe ATS pathway is primarily influenced by existing and announced policies and does not consider declining technology costs or significant new technologies.\n\n*Policy:* Unconditional national policies are the primary driver of transition outcomes in the ATS pathway. Energy efficiency targets and renewable energy targets, including detailed plans from national power development plans, drive significant renewables deployment and emissions reductions. Power capacity deployment follows the plans detailed in NDCs and PDPs until the final year laid out in these policies, and ATS simulates further additions based on the technology mix at the end year of the policy plans.\n\nThe ATS pathway assumes technology costs remain fixed at current levels across the full trajectory. The ATS pathway provides information on demand shifts, investment flows, and technology deployment, but these factors are driven primarily by modeled policy shifts. \n\n#### Application to Transition Assessment\n\nACE ATS is an exploratory, policy-focused pathway, which provides useful values for assessing the alignment of corporate plans to region-wide policy impacts. As a region-specific pathway, ACE ATS is not directly linked to a global implied temperature rise, limiting its use as a quantitative benchmark the ambition of climate targets.\n\nThe ATS is particularly well-suited for evaluating the alignment of corporate targets, strategies, and investment pipelines with the stated ambitions of the jurisdictions in which they operate. Its strong policy orientation allows for a meaningful comparison between corporate actions and national development priorities. A misalignment with the ATS may signal potential exposure to future regulatory risks, such as non-compliance with evolving energy policies or reduced competitiveness in markets where low-carbon technologies are being prioritized. Alignment, in the other hand, may indicate that a company is strategically positioned to benefit from market shifts. The ATS pathway includes the impacts of stated policies and targets which have not yet been implemented; therefore, misalignment does not necessarily imply exposure to current regulatory risk but rather a potential gap in future readiness. However, the ATS pathway excludes aspirational goals such as conditional NDCs, making the pathway a relatively conservative choice for evaluating potential policy alignment and impact.\n\nDue to its assumption of static technology costs and limited modelling of specific technology deployment, the ATS pathway has more limited applications in assessing the commercial or technological feasibility of specific transition strategies. Users interested in these applications may consider supplementing the ATS pathway with additional pathways that provide more detailed and dynamic technology cost and deployment projections.\n\nATS provides national targets. These targets are presented with fine granularity, enabling detailed national policy alignment assessments. The pathway also includes regional-level projections for generation and capacity on 5-year intervals and using a moderately detailed breakdown by energy sources such as coal, wind, solar, biomass, and geothermal. This data is relevant and useful for analyzing specific decarbonization levers and project investment pipelines within company plans, and supports region-specific benchmark, but does not enable country-specific comparisons of changes in generation or capacity.", + "pathwayDescription": "The ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE ASEAN Member States Targets Scenario (ATS) is an exploratory pathway which assumes that every ASEAN member state fully achieves its unconditional transition targets on time, including unconditional goals set via Nationally Determined Contributions (NDCs). This pathway models the impact of existing policies, such as national power development plans (PDPs), and the implementation of these stated targets. The ATS pathway models more ambitious energy efficiency targets, as well as a faster deployment of low-carbon energy technologies, especially from renewable energy. The ATS pathway does not include more aspirational national objectives, such as conditional NDCs or long-term net zero goals. For examples of pathways that include these policies, see the ACE Regional Aspiration Scenario (RAS) and IEA Announced Pledges Scenario (APS).\n\nPower generation emissions decline -2.4% per year, resulting in a total decrease of 74% relative to power emissions in the Baseline Scenario. However, the ATS pathway projects to an average annual emissions increase of 0.8% across the full energy sector.", "metric": [ "Emissions Intensity", "Capacity", @@ -81,22 +81,98 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Moderate increase", - "energyEfficiency": "Moderate improvement", - "energyDemand": "Significant increase", - "electrification": "Low or no change", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Moderate increase" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Moderate improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Significant increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], "policyTypes": [ - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "South East Asia", + "value": [ + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } ], - "technologyCostTrend": "Low or no change", - "emissionsScope": "CO2e (unspecified GHGs)", - "policyAmbition": "NDCs, unconditional only", - "technologyCostsDetail": "Capital costs, O&M, etc.", - "newTechnologiesIncluded": ["Green H2/ammonia", "SAF"], - "investmentNeeds": "By sector" - } + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "CO2e (unspecified GHGs)" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "NDCs, unconditional only" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Capital costs, O&M, etc." + } + ], + "newTechnologiesIncluded": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": ["Green H2/ammonia", "SAF"] + } + ], + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "By sector" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/asean-centre-for-energy/ACE-BAS-2024.json b/src/data/asean-centre-for-energy/ACE-BAS-2024.json index 1547ce38..e9138c70 100644 --- a/src/data/asean-centre-for-energy/ACE-BAS-2024.json +++ b/src/data/asean-centre-for-energy/ACE-BAS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "ACE-BAS-2024", "publication": { "title": { @@ -72,7 +72,7 @@ "technologies": [] } ], - "expertOverview": "#### Pathway Description\n\nThe ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE Baseline scenario (BAS) is a predictive pathway that extrapolates observed historic trends of the ASEAN Member States (AMS) energy systems into the future. It assumes a business-as-usual level of effort and specifically disregards modelling any policy interventions, even existing ones to meet national energy efficiency (EE) and renewable energy (RE) targets. Hence, it also excludes firm plant capacity additions based on power development plans (PDP). The extrapolation of past trends also implies that technology costs remain static and new technologies do not benefit from learning rates.\n\nAs a conservative baseline pathway, ACE BAS forecasts emissions at the ASEAN level to more than double by 2050.\n\n#### Core Drivers\n\nThe BAS pathway is driven by historical trends in energy consumption growth.\n\n*Economic growth:* Rising energy consumption in the BAS pathway is a function of region-wide population growth and rapid economic development. The total population in ASEAN is forecast to grow from 680 million in 2022 to 790 million in 2050. GDP growth across the region is projected to be 3.9% CAGR between that same period, along with further increases in electrification rate and clean cooking access.\n\nPolicy impacts are explicitly excluded from this model in order to provide a baseline case for modeling policy impacts, which are included in other ACE pathways such as the Regional Aspiration Scenario (RAS). Energy efficiency and technology costs are assumed to be static, leaving limited avenues for potential emissions reductions in this baseline pathway.\n\n#### Application to Transition Assessment\n\nThe ACE BAS pathway is intended to provide a base case for the ASEAN region, explicitly excluding policy impacts, technology cost declines, or efficiency improvements as avenues for emissions reductions. As a result, it provides a highly conversative reference, which may be used to place a floor on potential future rates of transition in the energy sector. As a single-region model, the BAS pathway is not associated with a global implied temperature rise, and should not be used as a quantitative benchmark for the ambition of climate targets.\n\nMisalignment to the baseline scenario can highlight companies or plans which do not keep pace with purely historical trends, without accounting for planned policy interventions. However, the scenario’s conservative limit its applications in assessing the ambition of corporate targets.\n\nFor transition assessment applications focused on assessing the commercial or technological feasibility of company plans, BAS’s assumptions of static technology costs and energy efficiency may be overly conservative compared to alternative business-as-usual pathways. Due to its deliberate exclusion of policy impacts, including existing policies, the BAS pathway should not be used for assessing the alignment of company strategies with existing or potential policies, and ACE provides alternate pathways for these applications.\n\nThe BAS provides benchmark data on 5-year intervals and uses a moderately detailed breakdown of specific generation technologies such as coal, wind, solar, biomass, and geothermal. This level of detail makes it well-suited to assessing specific decarbonization levers and project pipelines within company plans. BAS provides data at the regional ASEAN level, which enables region-specific benchmarking but limits its applicability for assessing country-specific strategies.", + "pathwayDescription": "The ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE Baseline scenario (BAS) is a predictive pathway that extrapolates observed historic trends of the ASEAN Member States (AMS) energy systems into the future. It assumes a business-as-usual level of effort and specifically disregards modelling any policy interventions, even existing ones to meet national energy efficiency (EE) and renewable energy (RE) targets. Hence, it also excludes firm plant capacity additions based on power development plans (PDP). The extrapolation of past trends also implies that technology costs remain static and new technologies do not benefit from learning rates.\n\nAs a conservative baseline pathway, ACE BAS forecasts emissions at the ASEAN level to more than double by 2050.", "metric": [ "Emissions Intensity", "Capacity", @@ -81,16 +81,92 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Significant increase", - "energyEfficiency": "Minor improvement", - "energyDemand": "Significant increase", - "electrification": "Low or no change", - "policyTypes": ["None"], - "technologyCostTrend": "Low or no change", - "emissionsScope": "CO2e (unspecified GHGs)", - "policyAmbition": "No policies included", - "technologyCostsDetail": "Capital costs, O&M, etc.", - "newTechnologiesIncluded": ["No new technologies"], - "investmentNeeds": "By sector" - } + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Significant increase" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Minor improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Significant increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], + "policyTypes": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": ["None"] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "CO2e (unspecified GHGs)" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "No policies included" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Capital costs, O&M, etc." + } + ], + "newTechnologiesIncluded": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": ["No new technologies"] + } + ], + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "By sector" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/asean-centre-for-energy/ACE-CNS-2024.json b/src/data/asean-centre-for-energy/ACE-CNS-2024.json index b3105738..c4e67c9f 100644 --- a/src/data/asean-centre-for-energy/ACE-CNS-2024.json +++ b/src/data/asean-centre-for-energy/ACE-CNS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "ACE-CNS-2024", "publication": { "title": { @@ -73,7 +73,7 @@ "technologies": [] } ], - "expertOverview": "#### Pathway Description\n\nThe ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The Carbon Neutral Scenario (CNS) is a normative pathway in which the ASEAN region achieves net-zero carbon emissions by 2050, both in energy and non-energy sectors. This pathway builds on the extensive set of stated and aspirational policies modelled in ACE’s Regional Aspiration Scenario (RAS), introducing further emissions constraints, accelerated low-emissions technology and low-carbon fuels availability, gradual retirement of some coal and gas technologies, and expanded low-emission power generation. It assumes that countries improve their energy efficiency according to their full potential, develop and deploy renewable energy sources according to their individual technical potential, and models capacity additions beyond existing power development plans (PDPs), prioritizing dispatch of renewable energy. The CNS pathway also models widespread adoption of carbon capture and storage (CCS) technology, and rapid scale-up of low-carbon fuels. The CNS pathway incorporates least-cost-optimization in its projections.\n\nCore Drivers\n\nThe CNS pathway combines widespread policy, rapid technology deployment, and explicit targets for emissions and emissions intensity reductions.\n\n*Policy:* The policies considered by CNS are equivalent to the RAS pathway, including nationally determined contributions, energy efficiency targets, and power development plans. These policies are a core driver of CNS projections but not differentiate it from the RAS pathway. \n\n*Emissions goals:* The CNS pathway introduces explicit emissions constraints, with specific emissions and emissions intensity reduction targets in power, industry, and transport beyond what is contained in existing policy and pledges. These constraints impose rapid emissions reductions across the full energy system. \n\n*Technology Deployment & Technology costs:* Emissions constraints and the least-cost approach led to a more rapid build-out of low-carbon power capacity than in other ACE pathways, including a more rapid deployment of technologies that are less mature or not yet established in the region (such as nuclear, geothermal, and tidal & wave power). However, the CNS pathway does not forecast changes in technology costs, limiting the potential impact of cost optimization. \n\nThe CNS pathway provides detailed information on demand and investment flows. The pathway forecasts very large increases in energy investment, with 55 billion USD a year between 2023 and 2030, and up to 371 bn USD between 2040-2050, more than twice the investment seen in the ACE RAS pathway.\n\n#### Application to Transition Assessment\n\nThe Carbon Neutrality Scenario is a normative pathway, which provides a useful trajectory for exploring rapid and ambitious transition across Southeast Asia. As a single-region model, CNS is not linked to a global emissions outcome and so cannot be used to quantify corporate ambition in terms of implied temperature rise – however, as a pathway with large scale emissions reductions by 2050, it can be used as a region-specific benchmark for high ambition strategies.\n\nAs a normative pathway, CNS includes significant interventions that exceed existing regional goals. As a result, (mis)alignment to the CNS pathway is less directly connected to regulatory risk or potential shifts in market share, as objectives within the CNS pathway may differ significantly from currently stated jurisdictional policies. Users interested in assessing the policy alignment or technological and commercial feasibility of corporate transition strategies should supplement CNS with pathways that provide more direct links to predicted policy and market conditions, such as ACE RAS and IEA APS.\n\nAs a cost-optimized pathway, the CNS pathway can be used to quantify investment needs associated with a rapid region-wide change and to contextualize the commercial and market feasibility of a corporate transition strategy. However, due to exclusion of technology cost changes over time in CNS, and users may wish to supplement it with options that provide more detailed and dynamic cost and deployment projections.\n\nThe CNS pathway includes regional-level projections for power generation and capacity on 5-year intervals and using a moderately detailed breakdown by energy sources such as coal, wind, solar, biomass, and geothermal. These are relevant and useful for analyzing specific decarbonization levers and project investment pipelines within company plans. However, CNS provides data at only the regional level. This limits its suitability for assessing companies which have operations concentrated in one or few countries, as it allows comparison only to the regional average.", + "pathwayDescription": "The ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The Carbon Neutral Scenario (CNS) is a normative pathway in which the ASEAN region achieves net-zero carbon emissions by 2050, both in energy and non-energy sectors. This pathway builds on the extensive set of stated and aspirational policies modelled in ACE’s Regional Aspiration Scenario (RAS), introducing further emissions constraints, accelerated low-emissions technology and low-carbon fuels availability, gradual retirement of some coal and gas technologies, and expanded low-emission power generation. It assumes that countries improve their energy efficiency according to their full potential, develop and deploy renewable energy sources according to their individual technical potential, and models capacity additions beyond existing power development plans (PDPs), prioritizing dispatch of renewable energy. The CNS pathway also models widespread adoption of carbon capture and storage (CCS) technology, and rapid scale-up of low-carbon fuels. The CNS pathway incorporates least-cost-optimization in its projections.", "metric": [ "Emissions Intensity", "Capacity", @@ -82,27 +82,98 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Moderate decrease", - "energyEfficiency": "Significant improvement", - "energyDemand": "Minor increase", - "electrification": "Moderate increase", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Moderate decrease" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Significant improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Minor increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Moderate increase" + } + ], "policyTypes": [ - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "South East Asia", + "value": [ + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "CO2e (unspecified GHGs)" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "High ambition policies" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Capital costs, O&M, etc." + } ], - "technologyCostTrend": "Low or no change", - "emissionsScope": "CO2e (unspecified GHGs)", - "policyAmbition": "High ambition policies", - "technologyCostsDetail": "Capital costs, O&M, etc.", "newTechnologiesIncluded": [ - "Battery storage", - "CCUS", - "Green H2/ammonia", - "SAF" + { + "sector": "across sectors", + "geography": "South East Asia", + "value": ["Battery storage", "CCUS", "Green H2/ammonia", "SAF"] + } ], - "investmentNeeds": "By sector" - } + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "By sector" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/asean-centre-for-energy/ACE-RAS-2024.json b/src/data/asean-centre-for-energy/ACE-RAS-2024.json index 4e57929e..360353e4 100644 --- a/src/data/asean-centre-for-energy/ACE-RAS-2024.json +++ b/src/data/asean-centre-for-energy/ACE-RAS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "ACE-RAS-2024", "publication": { "title": { @@ -72,7 +72,7 @@ "technologies": [] } ], - "expertOverview": "#### Pathway Description\n\nThe ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE Regional Aspiration Scenario (RAS) is an exploratory pathway which assumes that every ASEAN member state fully achieves both its unconditional and conditional transition targets on time, including Nationally Determined Contributions (NDCs), national power development plans (PDPs), and the enhanced scenarios of the national energy roadmaps of each ASEAN member state (e.g., the Clean Energy Scenario in the Philippines). The RAS pathway includes the 2025 ASEAN Plan of Action for Energy Cooperation (APAEC) targets for renewable energy generation and energy efficiency. The RAS pathway extends the ACE ATS pathway, covering all policies included in ATS with the inclusion of additional aspirational goals. The RAS also differs from the ATS pathway in using a least-cost optimization approach, which results in both a faster build-out of renewable energy and lower utilization rates for fossil fuel power plans. \n\nEmissions remain stable in the RAS until 2030, after which they start to decrease very slightly at a rate of approximately 0.1% per year across the energy sector. Emissions from power generation decrease much more rapidly to approximately a third of 2022 emissions by 2050. Emissions in the industry and residential sectors also drop significantly compared to the ATS.\n\n#### Core Drivers\n\nThe RAS pathway is primarily driven by a range of policy mechanisms and cost optimization based on current technology costs.\n\n*Policy:* The RAS pathway adds conditional NDC targets, national targets from the enhanced scenario of the AMS energy roadmap, and APAEC 2025 regional targets on top of existing policies and unconditional NDCS (modeled in ACE ATS). This increases the target share of renewable energy in total energy production and requires a steeper reduction of energy intensity in transport and industry.\n\n*Technology Costs and Technology Deployment:* RAS incorporates a least-cost approach to power capacity additions and generation. This results in a continued shift towards lower-cost renewables generation and a reduction in utilization rates for fossil fuel power plants. However, the RAS pathway does not include potential changes in technology costs over time. \n\nThe RAS pathway provides information on demand shifts, investment flows, and technology deployment, but these factors are driven primarily by modeled policy shifts.\n\n#### Application to Transition Assessment\n\nACE RAS is an exploratory, policy-focused pathway, which provides useful values for assessing the alignment of corporate plans to region-wide policy impacts under conditions of significant policy action. As a region-specific pathway, ACE RAS is not directly linked to a global implied temperature rise, limiting its use as a quantitative benchmark for the ambition of climate targets. However, for companies that aim to align their strategies with national policy targets, the RAS can be used to assess the level of alignment between company ambition and relevant policy ambition.\n\nThe RAS is designed to support alignment assessments with potential future policies, making it a valuable tool for evaluating corporate strategies and investment pipelines against both national and regional ambitions. A misalignment with the RAS may signal potential exposure to future regulatory risks, such as non-compliance with evolving energy policies or reduced competitiveness in markets where low-carbon technologies are being prioritized. Alignment may indicate that a company is strategically positioned to benefit from market shifts. The RAS pathway includes the impacts of aspirational policies and targets which have not yet been implemented; therefore, misalignment does not necessarily imply exposure to current regulatory risk but rather a potential gap in future readiness. Due to its inclusion of aspirational goals, alignment to the RAS pathway gives a stronger indication that a company is keeping pace with national ambition than alignment to a more limited policy pathway such as ACE ATS.\n\nDue to its inclusion of least-cost-optimized projections, the RAS pathway has useful applications for assessing the commercial feasibility of transition strategies. Alignment or misalignment to the RAS pathway may indicate that a company is outpacing or lagging economy-wide optimal trends. However, due to RAS’s assumption of static technology costs, users may consider supplementing the RAS pathway with options that provide more detailed and dynamic technology cost and deployment projections.\n\nThe RAS pathway provides regional-level projections for generation and capacity on 5-year intervals and using a moderately detailed breakdown by energy sources such as coal, wind, solar, biomass, and geothermal. This level of detail makes it well-suited to assessing specific decarbonization levers and project pipelines within company plans. However, RAS provides data at only the regional level. This limits its suitability for assessing companies which have operations concentrated in one or few countries, as it allows comparison only to the regional average.", + "pathwayDescription": "The ASEAN Centre for Energy (ACE) acts as the ASEAN energy data center and knowledge hub and produces the ACE ASEAN Energy Outlook providing energy pathways for Southeast Asia. The ACE Regional Aspiration Scenario (RAS) is an exploratory pathway which assumes that every ASEAN member state fully achieves both its unconditional and conditional transition targets on time, including Nationally Determined Contributions (NDCs), national power development plans (PDPs), and the enhanced scenarios of the national energy roadmaps of each ASEAN member state (e.g., the Clean Energy Scenario in the Philippines). The RAS pathway includes the 2025 ASEAN Plan of Action for Energy Cooperation (APAEC) targets for renewable energy generation and energy efficiency. The RAS pathway extends the ACE ATS pathway, covering all policies included in ATS with the inclusion of additional aspirational goals. The RAS also differs from the ATS pathway in using a least-cost optimization approach, which results in both a faster build-out of renewable energy and lower utilization rates for fossil fuel power plans. \n\nEmissions remain stable in the RAS until 2030, after which they start to decrease very slightly at a rate of approximately 0.1% per year across the energy sector. Emissions from power generation decrease much more rapidly to approximately a third of 2022 emissions by 2050. Emissions in the industry and residential sectors also drop significantly compared to the ATS.", "metric": [ "Emissions Intensity", "Capacity", @@ -81,22 +81,98 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Low or no change", - "energyEfficiency": "Significant improvement", - "energyDemand": "Moderate increase", - "electrification": "Low or no change", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Significant improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Moderate increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } + ], "policyTypes": [ - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "South East Asia", + "value": [ + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Low or no change" + } ], - "technologyCostTrend": "Low or no change", - "emissionsScope": "CO2e (unspecified GHGs)", - "policyAmbition": "NDCs incl. conditional targets", - "technologyCostsDetail": "Capital costs, O&M, etc.", - "newTechnologiesIncluded": ["Battery storage", "Green H2/ammonia", "SAF"], - "investmentNeeds": "By sector" - } + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "CO2e (unspecified GHGs)" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "NDCs incl. conditional targets" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "Capital costs, O&M, etc." + } + ], + "newTechnologiesIncluded": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": ["Battery storage", "Green H2/ammonia", "SAF"] + } + ], + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "South East Asia", + "value": "By sector" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/iea/IEA-APS-2024.json b/src/data/iea/IEA-APS-2024.json index 64bdfa8b..1037c645 100644 --- a/src/data/iea/IEA-APS-2024.json +++ b/src/data/iea/IEA-APS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "IEA-APS-2024", "publication": { "title": { @@ -316,8 +316,7 @@ "technologies": [] } ], - "pathwayOverview": "The Announced Pledges Scenario (APS) shows the future of the energy sector if all countries were to hit their aspirational targets, including national and regional net zero emissions pledges, on time and in full – in addition to their legislated policies (as described in STEPS). The APS has a 50% probability to not exceed 1.7°C by 2100. The difference between the APS and the Net Zero Emissions by 2050 scenario highlights the ambition gap between countries’ commitments and a 1.5°C pathway.", - "expertOverview": "#### Pathway Description\n\nThe Announced Pledges Scenario (APS) provides a pathway in which all countries achieve their aspirational energy transition goals, including national and regional net zero emissions pledges, in addition to their currently legislated policies. As an exploratory scenario, APS provides a detailed description of a possible future, but does not attempt to predict which announced policies will be implemented. The APS pathway corresponds to 1.7°C of warming by 2100 (50% probability). This is significantly lower than the IEA’s current policies scenario (STEPS), which forecasts 2.4°C of warming by 2100 (50% probability), but still higher than the IEA’s Net Zero scenario (NZE), which targets 1.5°C of warming by 2100 (50% probability). This indicates that the policies and pledges incorporated into APS significantly increase the pace of the transition compared to status quo conditions.\n\n#### Core Drivers\n\nIEA APS is primarily driven by large-scale policy action, continued cost decline for low-emissions technologies, and the introduction of new low-carbon technology.\n\n*Policy:* The APS assumes current nationally determined contributions (NDCs) are met, including, for example, emissions intensity goals and coal phase-out commitments. It includes national power development plans and targets for renewable energy additions or technology-based capacity mixes. In addition to explicit policies included in pledges, APS introduces additional measures to achieve aspirational policy goals. The APS models a carbon price, affecting electricity, industry and energy production sectors. The carbon tax is structured in three regional tiers: advanced economies, emerging markets and developing economies (EMDEs) with net-zero pledges, and EMDEs without pledges. \n\n*Technology costs:* IEA APS assumes significant cost declines in solar PV, onshore wind, and offshore wind power generation in most regions of the world and modest cost declines in nuclear power generation. These declines are mostly driven by reduced capital costs. The combined costs of fuel, CO2 prices, and operations and maintenance are projected to increase, making fossil-fuel based electricity less competitive over time.\n\nThe APS pathway provides detailed projections for demand shifts, infrastructure buildout, investment flow changes, and new technology deployment, but these factors are primarily caused by modelled policies and declining technology costs.\n\n#### Application to Transition Assessment\n\nAPS is a detailed, policy-focused, exploratory pathway. As a global integrated assessment model, APS provides a direct connection between the described pathway and a global temperature outcome. This makes it suitable for assessing the ambition of targets or plans against a 1.7°C outcome.\n\nDue to its strong policy focus, APS is well-suited to assessing the alignment of corporate targets, plans, and investment pipelines with the stated ambitions of the jurisdictions they operate in. Misalignment to the APS pathway can indicate potential risk that a company will fall out of line with future regulatory or economic policy and face declining market share as other segments of the energy sector are encouraged to grow. Conversely, alignment to APS may indicate that a company is well-positioned to take advantage of future policy and market dynamics. However, because APS includes aspirational pledges and policies which have not yet been implemented, misalignment to APS does not mean a company is automatically exposed to current regulatory risk.\n \nAPS provides benchmark data on 5- and 10-year intervals and uses a moderately detailed breakdown of specific generation technologies such as coal, onshore wind, offshore wind, solar PV, and geothermal. This level of detail makes it well-suited to assessing specific decarbonization levers and project pipelines within company plans. However, APS provides data at only the regional level, which limits its suitability for assessing companies which have operations concentrated in one or few countries, as it allows comparison only to the regional average.", + "pathwayDescription": "The Announced Pledges Scenario (APS) provides a pathway in which all countries achieve their aspirational energy transition goals, including national and regional net zero emissions pledges, in addition to their currently legislated policies. As an exploratory scenario, APS provides a detailed description of a possible future, but does not attempt to predict which announced policies will be implemented. The APS pathway corresponds to 1.7°C of warming by 2100 (50% probability). This is significantly lower than the IEA’s current policies scenario (STEPS), which forecasts 2.4°C of warming by 2100 (50% probability), but still higher than the IEA’s Net Zero scenario (NZE), which targets 1.5°C of warming by 2100 (50% probability). This indicates that the policies and pledges incorporated into APS significantly increase the pace of the transition compared to status quo conditions.", "metric": [ "Emissions Intensity", "Capacity", @@ -326,29 +325,99 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Moderate decrease", - "energyEfficiency": "Significant improvement", - "energyDemand": "Low or no change", - "electrification": "Moderate increase", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate decrease" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Low or no change" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate increase" + } + ], "policyTypes": [ - "Carbon price", - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "Global", + "value": [ + "Carbon price", + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Decrease" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "CO2" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "NDCs incl. conditional targets" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Capital costs, O&M, etc." + } ], - "technologyCostTrend": "Decrease", - "emissionsScope": "CO2", - "policyAmbition": "NDCs incl. conditional targets", - "technologyCostsDetail": "Capital costs, O&M, etc.", "newTechnologiesIncluded": [ - "CCUS", - "DAC", - "Green H2/ammonia", - "SAF", - "Battery storage" + { + "sector": "across sectors", + "geography": "Global", + "value": ["CCUS", "DAC", "Green H2/ammonia", "SAF", "Battery storage"] + } ], - "investmentNeeds": "By tech, part of value chain" - } + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "By tech, part of value chain" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/iea/IEA-NZE-2024.json b/src/data/iea/IEA-NZE-2024.json index 5aa1447e..20656946 100644 --- a/src/data/iea/IEA-NZE-2024.json +++ b/src/data/iea/IEA-NZE-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "IEA-NZE-2024", "publication": { "title": { @@ -98,7 +98,7 @@ "technologies": [] } ], - "expertOverview": "#### Pathway Description\n\nThe IEA Net Zero Emissions by 2050 (NZE) Scenario is a normative 1.5 aligned pathway, outlining how the global energy sector can reach net zero CO2 emissions by 2050. It assumes that advanced economies achieve net zero earlier (by mid 2040s), with emerging markets following by 2050. As a normative scenario, the NZE provides a detailed description of one pathway to reach a targeted future state, but it does not attempt to predict which policies or outcomes are most probable. The NZE pathway corresponds to a 50% probability of 1.5°C of warming by 2100. This is a more ambitious outcome than the IEA’s Announced Pledges Scenario (APS), which projects 1.7°C of warming by 2100 (50% probability), indicating a significant gap remains between announced governmental pledges and the trajectory described by the NZE pathway.\n\n#### Core Drivers\n\nThe NZE pathway is driven through a combination of wide-reaching policy, significant cost declines for existing green technologies, significant efficiency improvement and demand shifts, and the introduction of significant new technologies.\n\n*Policy:* The NZE pathway expands upon the policies considered in the APS pathway, most notably by introducing a more significant global carbon price for power, industry, and transport. A carbon price is introduced in all regions by 2030, with prices rising to USD$250/tCO2 in advanced economies, and USD$200/tCO2 in major emerging markets by 2050. Additionally, while the IEA NZE makes note of how the phase-out of fossil fuel subsidies need to be carefully designed to limit impacts on household budgets, they are largely removed in the NZE by 2030.\n\n*Technology costs:* The IEA NZE pathway assumes an S-curve trajectory of cost decline, with rapid early reductions as deployment scales up, followed by gradual flattening. This pattern applies across most renewables and transition technologies, including solar PV, offshore wind, battery EVs, and hydrogen fuel cells.\n\n*Technology shifts:* The NZE pathway is primarily driven by the rapid deployment of clean technology. Renewable energy capacity, led by solar PV and wind, approximately triples between 2023 and 2030, and reaches nearly 90% of global electricity by 2050. EVs account for approximately 60% of new car sales by 2030, and dominate the global fleet by 2050. Additionally, CCUS scales from 40Mt in 2023 to over 1Gt in 2030, and over 6Gt in 2050. This rapid expansion of clean technologies enables the peak of unabated fossil-fuel demand before 2030 and its subsequent decline by about 80% by 2050, marking the global phase down of coal, oil, and gas. \n\n*Falling energy demand:* The NZE pathway models significant energy efficiency improvements, alongside meaningful shifts in individual consumption patterns, which significantly reduce potential energy demand. Total energy consumption in the NZE pathway falls from well over 400 EJ in 2023 to under 350 EJ in 2050, a much larger decline than is observed in most similar pathways.\n\n#### Application to Transition Assessment\n\nIEA NZE is a global, temperature constrained normative scenario. As such, the NZE provides a direct connection between the described pathway and a global temperature outcome. It is well-suited for science-based target setting and net zero strategy alignment and is one of the most widely-used pathways to benchmark strategies against a projected 1.5°C temperature rise.\n\nAs a normative pathway, NZE models a potential pathway to achieve a target temperature outcome and introduces significant new policies and market shifts as part of the process. As a result, (mis)alignment to the NZE pathway is less directly connected to regulatory risk or potential shifts in market share, as objectives within the NZE pathway may differ significantly from currently stated jurisdictional goals. Users interested in assessing the policy alignment or technological and commercial feasibility of corporate transition strategies should review where underlying NZE assumptions diverge from current trends, and consider supplementing NZE with additional pathways that model a range of different policies and technology developments.\n\nThe NZE provides benchmark data on 5- and 10-year intervals and uses a moderately detailed breakdown of specific generation technologies such as coal, onshore wind, offshore wind, solar PV, and geothermal. This level of detail makes it well-suited to assessing specific decarbonization levers. However, NZE provides data only at the global level. This limits its suitability for assessing region- or jurisdiction-specific implications for company transition strategies.", + "pathwayDescription": "The IEA Net Zero Emissions by 2050 (NZE) Scenario is a normative 1.5 aligned pathway, outlining how the global energy sector can reach net zero CO2 emissions by 2050. It assumes that advanced economies achieve net zero earlier (by mid 2040s), with emerging markets following by 2050. As a normative scenario, the NZE provides a detailed description of one pathway to reach a targeted future state, but it does not attempt to predict which policies or outcomes are most probable. The NZE pathway corresponds to a 50% probability of 1.5°C of warming by 2100. This is a more ambitious outcome than the IEA’s Announced Pledges Scenario (APS), which projects 1.7°C of warming by 2100 (50% probability), indicating a significant gap remains between announced governmental pledges and the trajectory described by the NZE pathway.", "metric": [ "Emissions Intensity", "Capacity", @@ -107,29 +107,99 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Significant decrease", - "energyEfficiency": "Significant improvement", - "energyDemand": "Significant decrease", - "electrification": "Significant increase", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant decrease" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant decrease" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant increase" + } + ], "policyTypes": [ - "Carbon price", - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "Global", + "value": [ + "Carbon price", + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Decrease" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "CO2" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "High ambition policies" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Capital costs, O&M, etc." + } ], - "technologyCostTrend": "Decrease", - "emissionsScope": "CO2", - "policyAmbition": "High ambition policies", - "technologyCostsDetail": "Capital costs, O&M, etc.", "newTechnologiesIncluded": [ - "CCUS", - "DAC", - "Green H2/ammonia", - "SAF", - "Battery storage" + { + "sector": "across sectors", + "geography": "Global", + "value": ["CCUS", "DAC", "Green H2/ammonia", "SAF", "Battery storage"] + } ], - "investmentNeeds": "By tech, part of value chain" - } + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "By tech, part of value chain" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/iea/IEA-STEPS-2024.json b/src/data/iea/IEA-STEPS-2024.json index 414ae794..5f0a8bf3 100644 --- a/src/data/iea/IEA-STEPS-2024.json +++ b/src/data/iea/IEA-STEPS-2024.json @@ -1,5 +1,5 @@ { - "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", "id": "IEA-STEPS-2024", "publication": { "title": { @@ -316,8 +316,7 @@ "technologies": [] } ], - "pathwayOverview": "The Stated Policies Scenario (STEPS) provides a sense of the energy sector’s direction of travel today, based on the latest market data, technology costs and in-depth analysis of the prevailing policy settings in countries around the world. STEPS is complemented by APS, a scenario which assumes that countries’ aspirational targets will be met in full. The difference between STEPS and APS highlights a commitment gap.", - "expertOverview": "#### Pathway Description\n\nThe Stated Policies Scenario (STEPS) projects the energy sector’s current direction of travel, based on the latest market data, technology costs and in-depth analysis of the stated policies in countries around the world. STEPS provides a detailed projection of future outcomes based on current conditions, but does not attempt to predict which existing policies or economic conditions are more or less likely to change. The STEPS pathway corresponds to 2.4°C of warming by 2100 (50% probability). This is significantly higher than the IEA’s announced pledges scenario (APS), which projects 1.7°C of warming by 2100 (50% probability), and the IEA’s Net Zero scenario (NZE), which targets 1.5°C of warming by 2100 (50% probability).\n\n#### Core Drivers\n\nThe STEPS pathway is primarily driven by existing policy dynamics and forecast future declines in technology costs, which drive large-scale deployment of mature low-carbon technologies.\n\n*Policy:* The STEPS pathway models stated policies, such as energy subsidies or power development plans, but does not include aspirational policy goals that lack specific provisions for their implementation, such as long-term net-zero pledges. The STEPS pathway includes only existing or scheduled carbon pricing schemes for electricity, industry, and energy production sectors, and does not introduce a global carbon price, as is done in the APS and NZE pathway.\n\n*Technology costs:* Depending on the region, the IEA STEPS assumes moderate to significant cost declines in solar PV, onshore wind, and offshore wind power generation and modest cost declines in nuclear power generation. These declines are mostly driven by falling capital costs. The pathway projects moderate increases in fossil fuel power generation costs in advanced economies, but mixed trends in EMDEs. \n\nThe STEPS pathway provides detailed projections for demand shifts, infrastructure buildout, investment flows, and new technology deployment, but these factors are primarily caused by modelled existing policies and declining technology costs. \n\n#### Application to Transition Assessment\n\nAs a predictive pathway based on detailed modeling of current stated policies, market conditions, and technology trends, STEPS provides a valuable base-case projection for benchmarking the impact of future transition impacts in the absence of significant policy or market shifts. As a global integrated assessment model, STEPS provides a direct connection between the described pathway and a global temperature outcome. While this allows assessing the ambition of stated targets or plans against a 2.4°C outcome, this high level of implied temperature rise is not applicable for most institutional targets.\n\nDue to its strong focus on modeling existing policy and market trends, STEPS is well-suited to assessing the alignment of corporate targets, plans, and investment pipelines against a conservative forecast of market shifts. Misalignment to the STEPS pathway indicates that a company or plan may already be lagging forecasted rates of change, potentially exposing them to regulatory risks or declining market share as other segments of the energy sector grow. As STEPS only models stated policies with clear implementation plans, and does not include aspirational pledges such as those modeled in APS, it can be viewed as a conservative projection of potential market dynamics, and plans that exceed the rate of transition forecast in STEPS may still be exposed to future risks if new policies or technology shifts occur.\n\nSTEPS provides benchmark data on 5- and 10-year intervals and uses a moderately detailed breakdown of specific generation technologies such as coal, onshore wind, offshore wind, solar PV, and geothermal. This level of detail makes it well-suited to assessing specific decarbonization levers and project pipelines within company plans. However, STEPS provides data at only the regional level. This limits its suitability for assessing companies which have operations concentrated in one or few countries, as it allows comparison only to the regional average.", + "pathwayDescription": "The Stated Policies Scenario (STEPS) projects the energy sector’s current direction of travel, based on the latest market data, technology costs and in-depth analysis of the stated policies in countries around the world. STEPS provides a detailed projection of future outcomes based on current conditions, but does not attempt to predict which existing policies or economic conditions are more or less likely to change. The STEPS pathway corresponds to 2.4°C of warming by 2100 (50% probability). This is significantly higher than the IEA’s announced pledges scenario (APS), which projects 1.7°C of warming by 2100 (50% probability), and the IEA’s Net Zero scenario (NZE), which targets 1.5°C of warming by 2100 (50% probability).", "metric": [ "Emissions Intensity", "Capacity", @@ -326,29 +325,99 @@ "Absolute Emissions" ], "keyFeatures": { - "emissionsTrajectory": "Minor decrease", - "energyEfficiency": "Moderate improvement", - "energyDemand": "Moderate increase", - "electrification": "Moderate increase", + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Minor decrease" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate improvement" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate increase" + } + ], "policyTypes": [ - "Carbon price", - "Phaseout dates", - "Subsidies", - "Target technology shares", - "Performance standards", - "Other" + { + "sector": "across sectors", + "geography": "Global", + "value": [ + "Carbon price", + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Performance standards", + "Other" + ] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Decrease" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "CO2" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Current and drafted policies" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Capital costs, O&M, etc." + } ], - "technologyCostTrend": "Decrease", - "emissionsScope": "CO2", - "policyAmbition": "Current and drafted policies", - "technologyCostsDetail": "Capital costs, O&M, etc.", "newTechnologiesIncluded": [ - "CCUS", - "DAC", - "Green H2/ammonia", - "SAF", - "Battery storage" + { + "sector": "across sectors", + "geography": "Global", + "value": ["CCUS", "DAC", "Green H2/ammonia", "SAF", "Battery storage"] + } ], - "investmentNeeds": "By tech, part of value chain" - } + "investmentNeeds": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "By tech, part of value chain" + } + ] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] } diff --git a/src/data/pathwayMetadata.ts b/src/data/pathwayMetadata.ts index 14129fb9..d02ec5ab 100644 --- a/src/data/pathwayMetadata.ts +++ b/src/data/pathwayMetadata.ts @@ -1,7 +1,12 @@ import { PathwayMetadataType } from "../types"; import { FileEntry } from "../utils/validateData"; -import { assembleData, decideIncludeInvalid } from "../utils/loadData"; -import pathwayMetadataSchema from "../schema/pathwayMetadata.v1.json" with { type: "json" }; +import { + assembleData, + decideIncludeInvalid, + isViteDev, +} from "../utils/loadData"; +import pathwayMetadataSchema from "../schema/pathwayMetadata.v2.json" with { type: "json" }; +import pathwayMetadataV1Schema from "../schema/pathwayMetadata.v1.json" with { type: "json" }; import { commonSchemas } from "../schema/common"; // 1) Grab every JSON file in this folder **and subfolders** @@ -20,6 +25,40 @@ const entries: FileEntry[] = Object.entries(modules) })) .sort((a, b) => a.name.localeCompare(b.name)); +/** + * Count metadata files still carrying the v1 `$schema` (#858). + * + * `validateDataCollect` routes each document by its own `$schema` and drops + * anything that does not match the schema it was handed — as neither valid nor + * invalid. That is what lets v1 and v2 documents share src/data during the + * migration, but it also means an un-migrated file vanishes from the app with no + * error at all. Counting them here turns "pathways are missing" from a mystery + * into a number. Timeseries files are routed away by the same mechanism and are + * not counted, since their absence from this list is by design. + * + * Dev-server only. The message names an internal script and issue number, which + * is useful to us and noise to anyone reading a production console — and this + * module otherwise does no logging at all (assembleData warns only when a caller + * opts in via `opts.warn`, which this one does not). + */ +const V1_METADATA_ID = String( + (pathwayMetadataV1Schema as { $id?: string }).$id, +); + +const unmigrated = entries.filter( + (e) => + typeof e.data === "object" && + e.data !== null && + (e.data as { $schema?: unknown }).$schema === V1_METADATA_ID, +).length; + +if (unmigrated > 0 && isViteDev()) { + console.warn( + `[pathwayMetadata] ${unmigrated} metadata file(s) still use schema v1 and are ` + + `not loaded. Migrate them with scripts/codemod-v1-to-v2.ts (#858).`, + ); +} + export const pathwayMetadata: PathwayMetadataType[] = assembleData( entries, pathwayMetadataSchema, diff --git a/src/pages/ComparisonPage.test.tsx b/src/pages/ComparisonPage.test.tsx index 3c763a54..39044627 100644 --- a/src/pages/ComparisonPage.test.tsx +++ b/src/pages/ComparisonPage.test.tsx @@ -24,7 +24,11 @@ const fixtures = [ sectors: [{ name: "Power" }], metric: ["Capacity"], geography: { global: true, regions: { Europe: [] }, country: ["US"] }, - keyFeatures: { emissionsTrajectory: "foo" }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "across sectors", geography: "Global", value: "foo" }, + ], + }, }, { id: "cmp-b", @@ -41,7 +45,11 @@ const fixtures = [ sectors: [{ name: "Steel" }], metric: ["Generation"], geography: { country: ["DE", "FR"] }, - keyFeatures: { emissionsTrajectory: "bar" }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "across sectors", geography: "DE", value: "bar" }, + ], + }, }, ] as const; diff --git a/src/pages/PathwayDetailPage.tsx b/src/pages/PathwayDetailPage.tsx index e802cd90..42509a31 100644 --- a/src/pages/PathwayDetailPage.tsx +++ b/src/pages/PathwayDetailPage.tsx @@ -261,8 +261,17 @@ const PathwayDetailPage: React.FC = () => {

Expert Overview

+ {/* + v1's single `expertOverview` blob held three sections. In v2 + only the description survives as prose: "Core Drivers" becomes + the structured `coreDrivers` object, and "Application to + Transition Assessment" is retired because the new UI does not + display it (#898). So this renders one field, not a set of + sub-headed blocks. #859 owns the real presentation, including + whether this heading keeps its name. + */}
- {pathway.expertOverview} + {pathway.pathwayDescription ?? ""}
diff --git a/src/pages/PathwaySearch.test.tsx b/src/pages/PathwaySearch.test.tsx index 7a04b0e0..59bb751c 100644 --- a/src/pages/PathwaySearch.test.tsx +++ b/src/pages/PathwaySearch.test.tsx @@ -65,7 +65,17 @@ describe("PathwaySearch integration: dropdowns render and filter with 'None'", ( pathwayType: "Net Zero", modelYearNetzero: 2050, metric: [], - keyFeatures: { emissionsTrajectory: "foo" }, + keyFeatures: { + // "across regions", not a country: a pathway declaring no geography can + // name no place, so the sentinel is the only token it could carry. + emissionsTrajectory: [ + { + sector: "across sectors", + geography: "across regions", + value: "foo", + }, + ], + }, }, { id: "B", @@ -78,7 +88,11 @@ describe("PathwaySearch integration: dropdowns render and filter with 'None'", ( pathwayType: "Net Zero", modelYearNetzero: 2050, metric: ["Capacity"], - keyFeatures: { emissionsTrajectory: "foo" }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "across sectors", geography: "DE", value: "foo" }, + ], + }, }, { id: "C", @@ -89,7 +103,9 @@ describe("PathwaySearch integration: dropdowns render and filter with 'None'", ( pathwayType: "NZi2050", modelYearNetzero: 2040, metric: [], - keyFeatures: { emissionsTrajectory: "foo" }, + keyFeatures: { + emissionsTrajectory: [], // no geography declared -> no scope to attach a value to + }, }, { id: "D", @@ -100,7 +116,11 @@ describe("PathwaySearch integration: dropdowns render and filter with 'None'", ( pathwayType: "BAU", modelYearNetzero: 2030, metric: ["Capacity", "Generation"], - keyFeatures: { emissionsTrajectory: "bar" }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "across sectors", geography: "JP", value: "bar" }, + ], + }, }, { id: "E", @@ -111,7 +131,11 @@ describe("PathwaySearch integration: dropdowns render and filter with 'None'", ( pathwayType: "Net Zero", modelYearNetzero: 2050, metric: ["Generation"], - keyFeatures: { emissionsTrajectory: "bar" }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "across sectors", geography: "DE", value: "bar" }, + ], + }, }, ] as const; diff --git a/src/schema/common/dataAvailability.v1.json b/src/schema/common/dataAvailability.v1.json new file mode 100644 index 00000000..4d4a424d --- /dev/null +++ b/src/schema/common/dataAvailability.v1.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/common/dataAvailability.v1.json", + "title": "Data Availability", + "description": "The closed vocabularies describing how a metric's underlying data can be obtained (#870). Values follow cookbook tpr_cookbook_20260924; see docs/cookbook/split/data_availability in RMI/tpr-tpc.", + "$comment": "A wrapper object whose only job is to keep every $def reachable from the root: generate-types.ts runs json-schema-to-typescript with unreachableDefinitions:false, so a $def nothing points at emits no type. Same pattern as technology.v1.json and metric.v1.json. Callers $ref the individual $defs, not this object.", + "type": "object", + "properties": { + "timeResolution": { + "$ref": "#/$defs/timeResolution" + }, + "dataFormat": { + "$ref": "#/$defs/dataFormat" + }, + "granularityBreakdown": { + "$ref": "#/$defs/granularityBreakdown" + } + }, + "required": ["timeResolution", "dataFormat", "granularityBreakdown"], + "additionalProperties": false, + "$defs": { + "timeResolution": { + "description": "How finely the underlying data is resolved over time. Where a pathway resolves a metric differently for different geographies, the most granular resolution available is the one recorded.", + "$comment": "Spellings are the cookbook's literal allowed values, not abbreviations of them -- authored data comes from the classification workbook, whose validator enforces these exact strings. 'Unspecified' means the pathway does not say; 'Not covered' means this sector-metric pair is not covered at all, in which case every variable on the row reads 'Not covered' (enforced by validateScopedEntries).", + "type": "string", + "enum": [ + "2050 data point", + "Medium-term data point", + "10-year steps", + "5-year steps", + "1-year steps", + "5/10-year steps", + "Other time resolution", + "Unspecified", + "Not covered" + ] + }, + "dataFormat": { + "description": "In what form the source publication makes the metric's values available to a reader. Where values appear in more than one form the most extractable wins: Tabular before Text before Figure.", + "$comment": "Describes the source publication ONLY. Whether this repository hosts a copy is something it knows about itself -- derived from the generated timeseries index -- not a classification judgement about the pathway, which is why decision 0021 dropped the former 'In tool' member along with the separate 'access' (free/paywalled) field. There is deliberately no 'Unspecified' counterpart to timeResolution's: a metric whose values are visible at all has a visible form.", + "type": "string", + "enum": ["Tabular", "Text", "Figure", "Not covered"] + }, + "granularityBreakdown": { + "description": "Granularity values that are not technologies. A row's `granularity` is a flat array drawn from this vocabulary or from the sector's technologies, depending on the metric.", + "$comment": "The cookbook makes granularity conditional on (sector, metric): breakdown metrics take the sector's technologies, emissions metrics take an emissions scope, and the cost, investment, asset-lifetime, transmission and scrap-share metrics each take their own list. Those non-technology lists live here, flat, with per-metric membership left to the justification column rather than enforced -- the (sector, metric) conditional that would close it is the same draft-07 problem sectorSegment documents, and the lists do not overlap, so a value still identifies the metric it belongs to. The cost and investment lists are verbatim the keyFeatures `technologyCostsDetail` and `investmentNeeds` enums, which is the cookbook's own instruction: 'for any of the metrics whose allowed values replicate the allowed values of a key feature, apply the same assignment rules'. A sentinel must be the array's only member (enforced by validateScopedEntries).", + "type": "string", + "enum": [ + "No information", + "Unspecified", + "Not covered", + "Scope 1", + "Scope 1 & 2", + "Scope 1, 2 & 3", + "Scope 1 & 3", + "Total costs", + "Capital costs, O&M, etc.", + "Other cost breakdown", + "Total investment", + "By sector", + "By sector, part of value chain", + "By technology", + "By tech, part of value chain", + "High-carbon assets lifetime assumed constant", + "High-carbon assets lifetime face early retirement policy", + "International connections only", + "International and national transmission lines", + "Transmission and distribution", + "Scrap share as EAF input", + "Scrap share as steel total input" + ] + } + } +} diff --git a/src/schema/common/dataAvailabilityMetric.v1.json b/src/schema/common/dataAvailabilityMetric.v1.json new file mode 100644 index 00000000..e5b80ee4 --- /dev/null +++ b/src/schema/common/dataAvailabilityMetric.v1.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/common/dataAvailabilityMetric.v1.json", + "title": "Data Availability Metric", + "description": "The metric a dataAvailability row describes — the cookbook's 'Metrics – Extended' (#870).", + "$comment": "Deliberately NOT metric.v1.json. The cookbook carries two metric variables and register item D16 records that they legitimately differ rather than conflicting: metadata's `Metric` is the pathway-level multi-select, five values for Power, and drives the search facet; this one is the row key of the Data Availability table and adds the four metrics that are reported but not plotted (transmission lines, technology cost, investment requirement, asset lifetime). D16 also settles that `Storage capacity` is dropped, which the v2 model had already done. Capitalisation differs from metric.v1.json in places -- 'Emissions intensity' here against 'Emissions Intensity' there -- and #858 accepts that for now. Flat across sectors, with per-sector membership enforced by validateScopedEntries against SECTORS_BY_KEY in src/utils/timeseriesTaxonomy.ts, the same pattern sectorSegment and technology use and for the same draft-07 reason.", + "type": "object", + "properties": { + "displayName": { "$ref": "#/$defs/displayName" } + }, + "required": ["displayName"], + "additionalProperties": false, + + "$defs": { + "displayName": { + "description": "Display name of the metric as the Data Availability table keys its rows.", + "type": "string", + "enum": [ + "Emissions intensity", + "Absolute Emissions", + "Capacity", + "Generation", + "Technology mix", + "Transmission lines", + "Technology cost", + "Investment requirement", + "Asset lifetime", + "Emissions intensity (total)", + "Emissions intensity (primary)", + "Emissions intensity (secondary)", + "Absolute emissions", + "Technology mix (production by route)", + "Steel production by technology (production route)", + "Scrap share", + "Emissions intensity (passenger)", + "Emissions intensity (freight)", + "Absolute emissions Well-to-Wheel (passenger)", + "Absolute emissions Well-to-Wheel (freight)", + "Total demand (passenger)", + "Total demand (freight)", + "Demand by propulsion technology (passenger)", + "Demand by propulsion technology (freight)", + "Demand share by propulsion technology (passenger)", + "Demand share by propulsion technology (freight)" + ] + } + } +} diff --git a/src/schema/common/index.ts b/src/schema/common/index.ts index d7b9debc..966bdef0 100644 --- a/src/schema/common/index.ts +++ b/src/schema/common/index.ts @@ -28,6 +28,26 @@ export const geographySchema: SchemaObject = geographySchemaJson; import emissionsScopeSchemaJson from "./emissionsScope.v1.json" with { type: "json" }; export const emissionsScopeSchema: SchemaObject = emissionsScopeSchemaJson; +// The two axes of a v2 scoped keyFeatures entry (#858). +import scopeSectorSchemaJson from "./scopeSector.v2.json" with { type: "json" }; +export const scopeSectorSchema: SchemaObject = scopeSectorSchemaJson; + +import scopeGeographySchemaJson from "./scopeGeography.v2.json" with { type: "json" }; +export const scopeGeographySchema: SchemaObject = scopeGeographySchemaJson; + +// Vocabularies for the per-metric dataAvailability entries (#870). +import sectorSegmentSchemaJson from "./sectorSegment.v1.json" with { type: "json" }; +export const sectorSegmentSchema: SchemaObject = sectorSegmentSchemaJson; + +import dataAvailabilitySchemaJson from "./dataAvailability.v1.json" with { type: "json" }; +export const dataAvailabilitySchema: SchemaObject = dataAvailabilitySchemaJson; + +// Separate from metricSchema on purpose -- two metric variables, per register +// item D16. See dataAvailabilityMetric.v1.json's own $comment. +import dataAvailabilityMetricSchemaJson from "./dataAvailabilityMetric.v1.json" with { type: "json" }; +export const dataAvailabilityMetricSchema: SchemaObject = + dataAvailabilityMetricSchemaJson; + // Aggregate — type stays correct export const commonSchemas: SchemaObject[] = [ publicationSchema, @@ -39,6 +59,11 @@ export const commonSchemas: SchemaObject[] = [ countryCodeSchema, geographySchema, emissionsScopeSchema, + scopeSectorSchema, + scopeGeographySchema, + sectorSegmentSchema, + dataAvailabilitySchema, + dataAvailabilityMetricSchema, ]; export default commonSchemas; diff --git a/src/schema/common/scopeGeography.v2.json b/src/schema/common/scopeGeography.v2.json new file mode 100644 index 00000000..f376f91d --- /dev/null +++ b/src/schema/common/scopeGeography.v2.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "title": "Scope Geography", + "description": "The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label.", + "$comment": "Deliberately an open string rather than an enum. Region labels are free-form by design -- geography.v1's 'regions' keys are kept as named in the source publication -- so a closed list could not express labels this repo already uses, e.g. IEA's 'Asia Pacific', 'Eurasia', 'Middle East', 'Central and South America', none of which appear in geographyItem.v1's enum. (#858's scope text says geographyItem, which would reject any finer-scope entry on an IEA pathway.) 'Global' is the widest sentinel; 'across regions' is the cookbook's value for a multi-region aggregate with no single widest member (decision 0022), spelled as the cookbook spells it -- this repo previously used 'cross-region', a token the cookbook never defined, recorded as register item D15. Because any non-blank string validates here, the real constraint -- that the value is 'Global', 'across regions', or a region/country the pathway declares in its own 'geography' -- is enforced by validateScopedEntries in src/utils/validateScopes.ts, run from scripts/schema-check-files.ts. That check is what catches a mistyped label. Three-letter labels are allowed: an earlier `not` clause rejected every three-letter token, but the cookbook keeps a multi-country region's label exactly as the publication writes it, and model regions are often three letters (REMIND's SSA, LAM, EUR). The clause also guarded nothing the cross-field check does not: a stray ISO alpha-3 code such as 'THA' is rejected there unless the pathway declares it, because only declared labels, their alpha-2 members, declared countries, 'Global' and 'across regions' are allowed.", + "type": "string", + "minLength": 1, + "pattern": "^(?!\\s*$).+", + "examples": [ + "Global", + "across regions", + "South East Asia", + "Asia Pacific", + "TH", + "US" + ] +} diff --git a/src/schema/common/scopeSector.v2.json b/src/schema/common/scopeSector.v2.json new file mode 100644 index 00000000..9683bc55 --- /dev/null +++ b/src/schema/common/scopeSector.v2.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "title": "Scope Sector", + "description": "The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'.", + "$comment": "Members mirror sector.v1.json#/$defs/displayName, plus the 'across sectors' sentinel, spelled as the cookbook spells it (`key_features_sector_scope.md`; this repo previously used 'cross-sector', which register item D15 recorded as a plain enum mismatch). 'across sectors' means the union of THIS pathway's own declared sectors, not a universal match: an entry scoped 'across sectors' covers a query for sector X only when X is one of the sectors the pathway declares in its own 'sectors' array. So a pathway covering only Y and Z is NOT a match for a sector-X search, even though its 'across sectors' values would otherwise be broad enough. That constraint spans sibling data with dynamic keys, which draft-07 cannot express and AJV's $data cannot compute a union for, so it is enforced by validateScopedEntries in src/utils/validateScopes.ts, run from scripts/schema-check-files.ts.", + "type": "string", + "enum": [ + "across sectors", + "Land Use", + "Agriculture", + "Buildings", + "Steel", + "Cement", + "Chemicals", + "Coal Mining", + "Oil (Upstream)", + "Gas (Upstream)", + "Power", + "Automotive", + "Aviation", + "Rail", + "Shipping", + "Other" + ], + "examples": ["across sectors", "Power", "Steel"] +} diff --git a/src/schema/common/sectorSegment.v1.json b/src/schema/common/sectorSegment.v1.json new file mode 100644 index 00000000..f2d39373 --- /dev/null +++ b/src/schema/common/sectorSegment.v1.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/common/sectorSegment.v1.json", + "title": "Sector Segment", + "description": "Segments within a sector", + "type": "object", + "properties": { + "displayName": { + "$ref": "#/$defs/displayName" + } + }, + "required": ["displayName"], + "additionalProperties": false, + "$defs": { + "displayName": { + "description": "Display name of the sector segment as presented in tables.", + "$comment": "Segments are sector-specific: 'Energy storage' is meaningful for Power and meaningless for Land Use. draft-07 cannot condition this enum on a sibling 'sector' value -- and the if/then form that could collapses the generated object type to an index signature (see validateScopes.ts) -- so membership is enforced by validateScopedEntries against SECTORS_BY_KEY in src/utils/timeseriesTaxonomy.ts. Spellings follow cookbook data_availability/sector_segment_extended.md, which is why this is 'Transmission and distribution' rather than the '&' form used elsewhere in this repo. 'Unspecified' and 'Not covered' are legal under every sector, as is 'No information' -- the last has no cookbook counterpart and is kept pending review in RMI/tpr-tpc (#858).", + "type": "string", + "enum": [ + "No information", + "Unspecified", + "Not covered", + "Upstream energy and fuels", + "Passenger transport", + "Freight transport", + "Fuel extraction and processing", + "Power generation", + "Energy storage", + "Transmission and distribution", + "Mining", + "Ironmaking", + "Steelmaking", + "Downstream" + ] + } + } +} diff --git a/src/schema/common/technology.v1.json b/src/schema/common/technology.v1.json index bd8eb8f6..fc4230a3 100644 --- a/src/schema/common/technology.v1.json +++ b/src/schema/common/technology.v1.json @@ -5,11 +5,12 @@ "description": "Technologies for sectors", "type": "object", "properties": { - "displayName": { "$ref": "#/$defs/displayName" } + "displayName": { + "$ref": "#/$defs/displayName" + } }, "required": ["displayName"], "additionalProperties": false, - "$defs": { "displayName": { "description": "Display name of the technology as presented in charts or tables.", @@ -45,8 +46,29 @@ "Active Mobility", "Aviation Efficiency", "Maritime Efficiency", + "Geothermal", + "Energy storage", + "BOF", + "BF-BOF", + "BF-BOF+PCI", + "BF-BOF+CCUS", + "DRI-Melt-BOF", + "EAF", + "Scrap-EAF", + "DRI-EAF", + "DRI-EAF+H2", + "DRI-EAF+CCS", + "Electrolyser/Electrowinning", + "Jet Fuel", + "SAF", + "Electricity", + "Hydrogen", + "HEFA", + "PtL", + "AtJ", "Other" - ] + ], + "$comment": "Flat across sectors, with membership enforced per sector by technologyBelongsToSector against SECTORS_BY_KEY in src/utils/timeseriesTaxonomy.ts -- draft-07 cannot condition this enum on a sibling sector, and the if/then form that could collapses the generated object type to an index signature (see validateScopes.ts). Spellings follow cookbook metadata/technology_coverage.md. 'Renewables' and the pre-cookbook entries have no cookbook counterpart and are kept for backwards compatibility with v1 data (#858); they simply belong to no sector's list." } } } diff --git a/src/schema/pathwayMetadata.v1.json b/src/schema/pathwayMetadata.v1.json index f5daf1ad..d60af8ac 100644 --- a/src/schema/pathwayMetadata.v1.json +++ b/src/schema/pathwayMetadata.v1.json @@ -32,7 +32,7 @@ "tsType": "import('./common/publication.v1').PublicationV1" }, "pathwayType": { - "description": "Type of the pathway pathway.", + "description": "Type of the pathway.", "type": "string", "enum": ["Normative", "Exploratory", "Predictive"] }, diff --git a/src/schema/pathwayMetadata.v2.json b/src/schema/pathwayMetadata.v2.json new file mode 100644 index 00000000..6bd4ca72 --- /dev/null +++ b/src/schema/pathwayMetadata.v2.json @@ -0,0 +1,730 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", + "title": "Pathway Metadata", + "description": "A schema for the pathway metadata dataset in TPR.", + "type": "object", + "properties": { + "$schema": { + "description": "URI of the schema that validates this document (see https://json-schema.org/).", + "type": "string", + "maxLength": 1000 + }, + "id": { + "description": "The unique identifier for a pathway.", + "type": "string", + "maxLength": 100 + }, + "name": { + "description": "Name of the pathway.", + "$ref": "http://pathways.rmi.org/schema/common/label.v1.json", + "tsType": "import('./common/label.v1').LabelV1" + }, + "description": { + "description": "Brief description of the pathway.", + "type": "string", + "pattern": "\\.$", + "maxLength": 100 + }, + "publication": { + "description": "Bibliographic information about the report or dataset.", + "$ref": "http://pathways.rmi.org/schema/common/publication.v1.json", + "tsType": "import('./common/publication.v1').PublicationV1" + }, + "pathwayType": { + "description": "Type of the pathway.", + "type": "string", + "enum": ["Normative", "Exploratory", "Predictive"] + }, + "modelYearNetzero": { + "description": "Year by which net zero is reached in the pathway. If Pathway does not reach net zero, this field should be omitted.", + "type": "integer", + "minimum": 2030, + "maximum": 2100 + }, + "modelYearStart": { + "description": "Year from which the model starts.", + "type": "integer", + "minimum": 1900, + "maximum": 2030 + }, + "modelYearEnd": { + "description": "Year in which the model ends.", + "type": "integer", + "minimum": 2030, + "maximum": 2100 + }, + "modelTempIncrease": { + "description": "Modeled temperature increase expected by the pathway (in degrees Celsius).", + "type": "number", + "multipleOf": 0.1, + "minimum": 0.5, + "maximum": 3 + }, + "geography": { + "description": "Geographical areas that the pathway covers.", + "$ref": "http://pathways.rmi.org/schema/common/geography.v1.json", + "tsType": "import('./common/geography.v1').GeographyV1" + }, + "sectors": { + "description": "Sectors that the pathway covers.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "$ref": "http://pathways.rmi.org/schema/common/sector.v1.json#/$defs/displayName" + }, + "technologies": { + "type": "array", + "description": "Technologies applicable to this sector.", + "items": { + "$ref": "http://pathways.rmi.org/schema/common/technology.v1.json#/$defs/displayName" + } + }, + "segments": { + "description": "Segments of this sector's value chain that the pathway covers -- it provides at least one relevant output metric for each. Members must be segments of this entry's `name`, enforced by scripts/schema-check-files.ts.", + "$comment": "The cookbook's metadata `Sector segments` variable, which it writes sector-keyed: 'Power: [Power generation; Energy storage; Transmission and distribution]'. Nesting it inside `sectors[]` alongside `technologies` IS that keying, so no second mechanism is needed. Optional rather than required, unlike `technologies`: only three of the fifteen sectors have segments defined, and an absent list means 'not recorded' where an empty one would claim the pathway covers none. Distinct from dataAvailability's `byMetric[].sectorSegment`, which is per metric and can therefore be narrower than this.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "http://pathways.rmi.org/schema/common/sectorSegment.v1.json#/$defs/displayName", + "tsType": "import('./common/sectorSegment.v1').SectorSegmentV1['displayName']" + } + } + }, + "required": ["name", "technologies"], + "additionalProperties": false + } + }, + "pathwayDescription": { + "description": "Narrative description of the pathway. Replaces v1's expertOverview: in the v1 corpus this is the '#### Pathway Description' section of it. v1's separate pathwayOverview field is retired without replacement, not merged in here -- the two texts restate each other, so merging them read as immediate self-repetition. null means no description is available.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 2500 + }, + "metric": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "http://pathways.rmi.org/schema/common/metric.v1.json#/$defs/displayName", + "tsType": "import('./common/metric.v1').MetricV1['displayName']" + } + }, + "keyFeatures": { + "description": "Key features of the pathway. Every field is an array of {sector, geography, value} entries (#858), so a pathway can hold different values for different parts of its coverage. A non-varying feature carries exactly one entry at the widest applicable scope: sector 'across sectors' for a multi-sector pathway else its lone sector, and geography 'Global' else the pathway's widest declared region or country. An entry that is absent at some scope means the resolver keeps broadening until it finds one; an explicit \"No information\" value is a real authored value that terminates that fallback chain and displays at its own scope. An empty array means nothing is authored at any scope.", + "type": "object", + "properties": { + "emissionsTrajectory": { + "description": "Describes the overall trend of greenhouse gas emissions over time, from continued growth to rapid decline. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Significant increase", + "Moderate increase", + "Minor increase", + "Low or no change", + "Minor decrease", + "Moderate decrease", + "Significant decrease" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "energyEfficiency": { + "description": "Indicates how efficiently energy is used to produce economic output across the sectors covered in the pathway. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Significant deterioration", + "Moderate deterioration", + "Minor deterioration", + "Low or no change", + "Minor improvement", + "Moderate improvement", + "Significant improvement", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "energyDemand": { + "description": "Captures the change in total energy consumption, driven by factors such as socio-economic development, technology shifts and consumer behavior. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Significant decrease", + "Moderate decrease", + "Minor decrease", + "Low or no change", + "Minor increase", + "Moderate increase", + "Significant increase" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "electrification": { + "description": "Represents the extent to which energy end-uses transition from fossil fuels to electricity. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Significant decrease", + "Moderate decrease", + "Minor decrease", + "Low or no change", + "Minor increase", + "Moderate increase", + "Significant increase", + "Not Applicable", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "policyTypes": { + "description": "Identifies the types of policies modeled as drivers of the pathway, such as carbon pricing, subsidies, or mandated phaseouts of specific technologies. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "No information", + "Carbon price", + "Feed-in tariffs", + "Performance standards", + "Phaseout dates", + "Subsidies", + "Target technology shares", + "Other", + "None", + "Not applicable at this scope level" + ] + } + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "technologyCostTrend": { + "description": "Describes how technology costs evolve over time, from static cost assumptions to rapidly declining costs (e.g., via learning curves). Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Increase", + "Low or no change", + "Decrease", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "emissionsScope": { + "description": "Defines which greenhouse gases are covered in the pathway's modeled emissions. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "$comment": "An anyOf rather than an appended enum member, unlike the other eight fields that take the scope sentinel. This field's vocabulary lives in emissionsScope.v1.json, which pathwayTimeseries.v1 and pathwayMetadata.v1 also $ref -- and 'Not applicable at this scope level' is a statement about a keyFeatures row's scope, meaningless on a timeseries. Composing here keeps the shared list the single source of the gas vocabulary while adding the sentinel only where it means something.", + "anyOf": [ + { + "$ref": "http://pathways.rmi.org/schema/common/emissionsScope.v1.json", + "tsType": "import('./common/emissionsScope.v1').EmissionsScopeV1" + }, + { + "type": "string", + "enum": ["Not applicable at this scope level"] + } + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "policyAmbition": { + "description": "Represents the overall stringency and intent of modeled policies relative to climate targets, often reflecting if and how far the included policies go beyond currently legislated ones. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "No policies included", + "Current/legislated policies", + "Current and drafted policies", + "NDCs, unconditional only", + "NDCs incl. conditional targets", + "High ambition policies", + "Other policy ambition", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "technologyCostsDetail": { + "description": "Specifies the level of granularity in cost data, such as total system costs or detailed CAPEX/OPEX breakdowns. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Total costs", + "Capital costs, O&M, etc.", + "Other cost breakdown", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "newTechnologiesIncluded": { + "description": "Lists emerging or breakthrough technologies that are explicitly modeled within the pathway. These are considered in technology deployment too. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "No information", + "No new technologies", + "CCUS", + "DAC", + "Green H2/ammonia", + "SAF", + "Battery storage", + "EGS/AGS", + "SMR", + "Other new technologies", + "Not applicable at this scope level" + ] + } + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + }, + "investmentNeeds": { + "description": "Summarizes how investment requirements are quantified, from total system to sector-level or supply-chain detail. Scoped: see keyFeatures.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "sector": { + "$ref": "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + "tsType": "import('./common/scopeSector.v2').ScopeSectorV2" + }, + "geography": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + }, + "value": { + "type": "string", + "enum": [ + "No information", + "Total investment", + "By sector", + "By sector, part of value chain", + "By technology", + "By tech, part of value chain", + "Not applicable at this scope level" + ] + } + }, + "required": ["sector", "geography", "value"], + "additionalProperties": false + } + } + }, + "additionalProperties": false, + "required": [ + "emissionsTrajectory", + "energyEfficiency", + "energyDemand", + "electrification", + "policyTypes", + "technologyCostTrend", + "emissionsScope", + "policyAmbition", + "technologyCostsDetail", + "newTechnologiesIncluded", + "investmentNeeds" + ] + }, + "coreDrivers": { + "description": "The drivers that shape this pathway's outcomes. Every field is required but nullable: null means the driver is not a core driver for this pathway, as distinct from a driver that is present but undescribed.", + "type": "object", + "properties": { + "policies": { + "description": "Policies modeled as a driver of this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "emissionsTargets": { + "description": "Emissions targets or constraints driving this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "technologyCosts": { + "description": "Technology cost assumptions driving this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "investmentChange": { + "description": "Changes in investment driving this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "macroeconomicDrivers": { + "description": "Macroeconomic assumptions driving this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "behavioralShifts": { + "description": "Behavioral or demand-side shifts driving this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + }, + "otherDrivers": { + "description": "Any other core driver of this pathway.", + "type": ["string", "null"], + "pattern": "\\.$", + "maxLength": 500 + } + }, + "additionalProperties": false, + "required": [ + "policies", + "emissionsTargets", + "technologyCosts", + "investmentChange", + "macroeconomicDrivers", + "behavioralShifts", + "otherDrivers" + ] + }, + "dependencies": { + "description": "Conditions the pathway's outcomes depend on. Descriptive only -- deliberately NOT part of the #869 inheritance chain, so these are not scoped by geography.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "dependency_name": { + "description": "Category of the dependency.", + "type": "string", + "enum": [ + "Policy strategy", + "Regulatory framework", + "Market and economics", + "Public acceptance", + "Consumer and client behavior", + "Infrastructure and logistics", + "Technology", + "Resource availability", + "Environmental impacts and ecosystem services", + "Labor availability" + ] + }, + "dependency_description": { + "description": "What the pathway depends on, in prose.", + "type": "string", + "pattern": "\\.$", + "maxLength": 500 + }, + "sector": { + "description": "Sector the dependency applies to. Must be one of the pathway's own declared sectors -- enforced by scripts/schema-check-files.ts, since draft-07 cannot reference sibling data.", + "$ref": "http://pathways.rmi.org/schema/common/sector.v1.json#/$defs/displayName" + }, + "evidence_type": { + "description": "Strength of the evidence for this dependency.", + "type": "string", + "enum": ["Quantitative", "Qualitative", "Anecdotal", "No evidence"] + } + }, + "required": [ + "dependency_name", + "dependency_description", + "sector", + "evidence_type" + ], + "additionalProperties": false + } + }, + "dataAvailability": { + "description": "Where and how the data behind each metric can be obtained (#870). Optional: authoring is incremental, and a pathway with no entry yet is not an invalid pathway. Absent means unknown, NOT unavailable -- a row's `Not covered` values say unavailable.", + "type": "object", + "properties": { + "overall": { + "description": "Summary of the full timeseries file we host, plus anything that does not fit the per-metric rows. Null where there is nothing to add.", + "type": ["string", "null"], + "maxLength": 2500 + }, + "byMetric": { + "description": "One row per (metricName, sector, sectorSegment, geography set). Each combination may appear only once -- enforced by scripts/schema-check-files.ts, because uniqueItems compares whole entries and so permits two rows that agree on the scope and disagree on everything else. The cookbook expects a row for every allowable (sector, metric) pair in a covered sector, with an uncovered pair recorded as 'Not covered' rather than omitted.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "metricName": { + "description": "Metric this row describes -- the cookbook's 'Metrics - Extended', which is a different vocabulary from the pathway-level `metric` field (register item D16). Must be a metric of this row's `sector`, enforced by scripts/schema-check-files.ts.", + "$ref": "http://pathways.rmi.org/schema/common/dataAvailabilityMetric.v1.json#/$defs/displayName", + "tsType": "import('./common/dataAvailabilityMetric.v1').DataAvailabilityMetricV1['displayName']" + }, + "sector": { + "description": "Sector this row describes. Must be one of the pathway's own declared sectors. Note this is the plain sector enum, not scopeSector.v2: there is no `across sectors` availability, because availability is a property of a concrete dataset.", + "$ref": "http://pathways.rmi.org/schema/common/sector.v1.json#/$defs/displayName", + "tsType": "import('./common/sector.v1').SectorV1['displayName']" + }, + "sectorSegment": { + "description": "Segments of this row's sector that the metric covers. `Unspecified` where the pathway names none, `Not covered` where the sector-metric pair is not covered; either must then be the only member. Members must be segments of this row's `sector`, enforced by scripts/schema-check-files.ts.", + "$comment": "An array because the cookbook types `Sector segment - Extended` as Multiple, and real rows use it: the gold set has `Power generation; Energy storage` and `Steel: [Ironmaking, Steelmaking]` in single cells. Assignment is per sector AND metric, so coverage varies between metrics of the same sector.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "http://pathways.rmi.org/schema/common/sectorSegment.v1.json#/$defs/displayName", + "tsType": "import('./common/sectorSegment.v1').SectorSegmentV1['displayName']" + } + }, + "geography": { + "description": "Geographies this row covers, as the cookbook's `Geography coverage`: a subset of the pathway's own declared geography, enforced by scripts/schema-check-files.ts. Same scope tokens as keyFeatures, so the table can be filtered by the detail page's geography selection (#872) using the existing scope helpers. `Unspecified` where the pathway does not say which, `Not covered` where the sector-metric pair is not covered; either must then be the only member.", + "$comment": "An array because the cookbook types this variable as Multiple -- one metric may be projected for Global and for individual countries at once ('Global; Western Europe: [FR, ES, PT]; JP: [JP]'). It absorbed the former single-valued `geography` and the separate `geographyCoverage` class enum (Global/Regional/Country), which had no cookbook counterpart and duplicated, less precisely, what the token list already says.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + "tsType": "import('./common/scopeGeography.v2').ScopeGeographyV2" + } + }, + "timeResolution": { + "description": "How finely the underlying data is resolved over time.", + "$ref": "http://pathways.rmi.org/schema/common/dataAvailability.v1.json#/$defs/timeResolution" + }, + "dataFormat": { + "description": "In what form the source publication reports this metric's values.", + "$ref": "http://pathways.rmi.org/schema/common/dataAvailability.v1.json#/$defs/dataFormat" + }, + "granularity": { + "description": "Dimensions the metric is broken down by. Members are either technologies of this row's sector -- enforced by scripts/schema-check-files.ts, the same rule as sectors[].technologies -- or values from the granularityBreakdown vocabulary. `Unspecified` where the pathway does not clarify the breakdown, `Not covered` where the sector-metric pair is not covered; either must then be the only member.", + "$comment": "No longer nullable: the cookbook distinguishes 'the pathway does not say' (Unspecified) from 'this pair is not covered' (Not covered), a distinction a bare null cannot carry. Same authored-absence rule as keyFeatures' explicit 'No information'.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "anyOf": [ + { + "$ref": "http://pathways.rmi.org/schema/common/technology.v1.json#/$defs/displayName", + "tsType": "import('./common/technology.v1').TechnologyV1['displayName']" + }, + { + "$ref": "http://pathways.rmi.org/schema/common/dataAvailability.v1.json#/$defs/granularityBreakdown" + } + ] + } + }, + "scopeLimitations": { + "description": "Caveats on what the data does and does not cover, in prose: the boundaries of the sector model, what is in and out of scope, and for emissions metrics the emissions scope. `Unspecified` where the pathway does not say, `Not covered` where the sector-metric pair is not covered.", + "$comment": "No longer nullable, for the same reason as granularity. Where the pathway reports storage, the cookbook requires this field to state which storage technologies the figure covers -- 'Energy storage' is a single value carrying no technology detail, so this note is the only place the breakdown survives.", + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "metricName", + "sector", + "sectorSegment", + "geography", + "timeResolution", + "dataFormat", + "granularity", + "scopeLimitations" + ], + "additionalProperties": false + } + } + }, + "required": ["overall", "byMetric"], + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": [ + "id", + "name", + "description", + "publication", + "pathwayType", + "geography", + "sectors", + "pathwayDescription", + "metric", + "keyFeatures", + "coreDrivers", + "dependencies" + ] +} diff --git a/src/schema/pathwayMetadata.v2.test.ts b/src/schema/pathwayMetadata.v2.test.ts new file mode 100644 index 00000000..a617a588 --- /dev/null +++ b/src/schema/pathwayMetadata.v2.test.ts @@ -0,0 +1,514 @@ +import { describe, it, expect } from "vitest"; +import v1Json from "./pathwayMetadata.v1.json" with { type: "json" }; +import v2Json from "./pathwayMetadata.v2.json" with { type: "json" }; +import scopeSectorJson from "./common/scopeSector.v2.json" with { type: "json" }; +import sectorJson from "./common/sector.v1.json" with { type: "json" }; +import emissionsScopeJson from "./common/emissionsScope.v1.json" with { type: "json" }; +import dataAvailabilityJson from "./common/dataAvailability.v1.json" with { type: "json" }; + +/** + * Guards v2's keyFeatures against silent self-drift. + * + * v2 spells the scoped-entry wrapper out once per field rather than sharing a + * `$defs` entry via `allOf`. That was measured, not assumed: the `allOf` version + * validates identically and generates nicer types, but AJV's `strict: true` + * (`strictRequired`, then `strictTypes`) forces the boilerplate back in for a net + * saving of 18 lines, it cannot use `additionalProperties: false` — that keyword + * only sees its own branch's `properties`, so it would reject `value` — and the + * `propertyNames` substitute degrades the commonest authoring error from + * "must NOT have additional properties" to "property name must be valid" with the + * offending key unnamed, because `fmt()` in validateData.ts drops AJV's `params`. + * + * The cost of that choice is 11 copies of one shape, so these tests enforce what + * the `$ref` would have: that the copies stay identical, and that each field's + * `value` still matches v1's enum verbatim, which is #858's actual requirement. + */ + +/** The slice of JSON Schema draft-07 these assertions actually read. */ +interface JsonSchema { + $id?: string; + $ref?: string; + $defs?: Record; + type?: string | string[]; + enum?: string[]; + items?: JsonSchema; + anyOf?: JsonSchema[]; + properties?: Record; + required?: string[]; + additionalProperties?: boolean; + uniqueItems?: boolean; + minItems?: number; + minLength?: number; + maxLength?: number; + description?: string; + tsType?: string; +} + +const v1 = v1Json as unknown as JsonSchema; +const v2 = v2Json as unknown as JsonSchema; +const scopeSector = scopeSectorJson as unknown as JsonSchema; +const sector = sectorJson as unknown as JsonSchema; +const emissionsScope = emissionsScopeJson as unknown as JsonSchema; +const dataAvailability = dataAvailabilityJson as unknown as JsonSchema; + +/** Throwing accessors keep every read type-safe without non-null assertions. */ +function props(schema: JsonSchema, where: string): Record { + if (!schema.properties) throw new Error(`${where}: expected properties`); + return schema.properties; +} + +function prop(schema: JsonSchema, name: string, where: string): JsonSchema { + const found = props(schema, where)[name]; + if (!found) throw new Error(`${where}: expected property ${name}`); + return found; +} + +function items(schema: JsonSchema, where: string): JsonSchema { + if (!schema.items) throw new Error(`${where}: expected items`); + return schema.items; +} + +function enumOf(schema: JsonSchema, where: string): string[] { + if (!schema.enum) throw new Error(`${where}: expected enum`); + return schema.enum; +} + +const KEY_FEATURE_FIELDS = [ + "emissionsTrajectory", + "energyEfficiency", + "energyDemand", + "electrification", + "policyTypes", + "technologyCostTrend", + "emissionsScope", + "policyAmbition", + "technologyCostsDetail", + "newTechnologiesIncluded", + "investmentNeeds", +] as const; + +/** v1 stores these as arrays, so in v2 the whole array is one entry's `value`. */ +const ARRAY_VALUED: ReadonlySet = new Set([ + "policyTypes", + "newTechnologiesIncluded", +]); + +/** v1's only field whose enum lacked "No information" — it has "None" instead. */ +const GAINED_NO_INFORMATION = "policyTypes"; + +/** + * The scope sentinel, and the fields that do *not* take it. + * + * The cookbook's rule (`key_features/00_key_features_notes.md`) is "every key + * feature that is `widest only` on either axis". Its cardinality table makes + * `emissionsTrajectory` and `energyDemand` the only two that are + * `widest + all in-scope` on both, so they are the only two without it — which + * is why this is written as the exception list rather than the nine. + */ +const SCOPE_SENTINEL = "Not applicable at this scope level"; +const WIDEST_ON_BOTH_AXES: ReadonlySet = new Set([ + "emissionsTrajectory", + "energyDemand", +]); + +/** + * Values v2 adds to a field beyond the scope sentinel, with the cookbook's + * reason. Listed rather than tolerated, so an unplanned addition still fails. + */ +const DELIBERATE_ADDITIONS: Readonly> = { + // A property of the sector (meaningless for Power), not of the row's scope, + // so it survives at every scope where the sentinel would not. + electrification: ["Not Applicable"], + // Cookbook decision 0020. + newTechnologiesIncluded: ["SMR"], +}; + +/** What v2's enum should hold, given v1's, for one field. */ +function expectedV2Members( + name: string, + v1Members: readonly string[], +): string[] { + const members = [...v1Members]; + if (name === GAINED_NO_INFORMATION) members.unshift("No information"); + for (const added of DELIBERATE_ADDITIONS[name] ?? []) { + // SMR sits before "Other …", matching how every other enum keeps its + // catch-all last; the rest append. + const other = members.findIndex((m) => m.startsWith("Other ")); + if (added === "SMR" && other !== -1) members.splice(other, 0, added); + else members.push(added); + } + if (!WIDEST_ON_BOTH_AXES.has(name)) members.push(SCOPE_SENTINEL); + return members; +} + +const kf2 = prop(v2, "keyFeatures", "v2"); +const kf1 = prop(v1, "keyFeatures", "v1"); + +/** The v2 field schema (the array), and the scoped entry inside it. */ +const field = (name: string): JsonSchema => prop(kf2, name, "v2.keyFeatures"); +const entry = (name: string): JsonSchema => + items(field(name), `v2.keyFeatures.${name}`); +const entryValue = (name: string): JsonSchema => + prop(entry(name), "value", `v2.keyFeatures.${name}.items`); + +describe("pathwayMetadata.v2 sectors[].segments (#858)", () => { + const sectorEntry = items(prop(v2, "sectors", "v2"), "v2.sectors"); + + it("nests segments inside sectors[] rather than keying them separately", () => { + // The cookbook writes this variable sector-keyed -- "Power: [Power + // generation; Energy storage]" -- and sitting inside sectors[] alongside + // technologies *is* that keying, so there is no second mechanism. + expect(Object.keys(props(sectorEntry, "v2.sectors.items")).sort()).toEqual([ + "name", + "segments", + "technologies", + ]); + const segments = prop(sectorEntry, "segments", "v2.sectors.items"); + expect(segments.type).toBe("array"); + expect(segments.uniqueItems).toBe(true); + expect(items(segments, "segments").$ref).toBe( + "http://pathways.rmi.org/schema/common/sectorSegment.v1.json#/$defs/displayName", + ); + }); + + it("leaves segments optional, unlike technologies", () => { + // Only three of fifteen sectors have segments defined, and an absent list + // means "not recorded" where [] would claim the pathway covers none. + expect([...(sectorEntry.required ?? [])].sort()).toEqual([ + "name", + "technologies", + ]); + }); +}); + +describe("pathwayMetadata.v2 keyFeatures — field set", () => { + it("declares exactly the 11 fields, and the same ones as v1", () => { + expect(Object.keys(props(kf2, "v2.keyFeatures"))).toEqual([ + ...KEY_FEATURE_FIELDS, + ]); + expect(Object.keys(props(kf2, "v2.keyFeatures"))).toEqual( + Object.keys(props(kf1, "v1.keyFeatures")), + ); + }); + + it("requires all 11 and forbids extras", () => { + expect([...(kf2.required ?? [])].sort()).toEqual( + [...KEY_FEATURE_FIELDS].sort(), + ); + expect(kf2.additionalProperties).toBe(false); + }); +}); + +describe("pathwayMetadata.v2 keyFeatures — the wrapper is identical everywhere", () => { + it.each(KEY_FEATURE_FIELDS)( + "%s is a uniqueItems array with a description", + (name) => { + const f = field(name); + expect(f.type).toBe("array"); + expect(f.uniqueItems).toBe(true); + // No minItems: an empty array is the legal "absent at every scope" state. + expect(f.minItems).toBeUndefined(); + expect(typeof f.description).toBe("string"); + expect(f.description?.endsWith(".")).toBe(true); + }, + ); + + it.each(KEY_FEATURE_FIELDS)( + "%s entries are closed {sector, geography, value}", + (name) => { + const e = entry(name); + expect(e.type).toBe("object"); + expect(Object.keys(props(e, name)).sort()).toEqual([ + "geography", + "sector", + "value", + ]); + expect([...(e.required ?? [])].sort()).toEqual([ + "geography", + "sector", + "value", + ]); + expect(e.additionalProperties).toBe(false); + }, + ); + + it("every field's sector and geography subschemas are byte-identical", () => { + // The whole point of the guard: one field drifting is the failure mode a + // shared $ref would have made impossible. + const scopes = KEY_FEATURE_FIELDS.map((name) => + JSON.stringify({ + sector: prop(entry(name), "sector", name), + geography: prop(entry(name), "geography", name), + }), + ); + expect(new Set(scopes).size).toBe(1); + }); + + it("points sector and geography at the v2 scope subschemas", () => { + const e = entry("emissionsTrajectory"); + expect(prop(e, "sector", "sector").$ref).toBe( + "http://pathways.rmi.org/schema/common/scopeSector.v2.json", + ); + expect(prop(e, "geography", "geography").$ref).toBe( + "http://pathways.rmi.org/schema/common/scopeGeography.v2.json", + ); + // Without the tsType hints the generator inlines the unions per field + // instead of importing the named types. + expect(prop(e, "sector", "sector").tsType).toContain("ScopeSectorV2"); + expect(prop(e, "geography", "geography").tsType).toContain( + "ScopeGeographyV2", + ); + }); +}); + +describe("pathwayMetadata.v2 keyFeatures — values carry over from v1", () => { + it.each(KEY_FEATURE_FIELDS)("%s value enum matches v1", (name) => { + const v2Value = entryValue(name); + const v1Value = prop(kf1, name, "v1.keyFeatures"); + + if (name === "emissionsScope") { + // v1 is a bare $ref to the shared gas vocabulary. v2 composes that same + // $ref with the scope sentinel, rather than appending a member to the + // common schema, which pathwayTimeseries.v1 also $refs — see the + // field's own $comment. + const branches = v2Value.anyOf ?? []; + expect(branches).toHaveLength(2); + expect(branches[0].$ref).toBe(v1Value.$ref); + expect(enumOf(branches[1], "emissionsScope sentinel")).toEqual([ + SCOPE_SENTINEL, + ]); + return; + } + + if (ARRAY_VALUED.has(name)) { + // The v1 array becomes one entry's value, not one entry per member. + expect(v2Value.type).toBe("array"); + expect(v2Value.uniqueItems).toBe(v1Value.uniqueItems); + expect(v2Value.minItems).toBe(v1Value.minItems); + const v1Members = enumOf(items(v1Value, name), name); + expect(enumOf(items(v2Value, name), name)).toEqual( + expectedV2Members(name, v1Members), + ); + return; + } + + expect(v2Value.type).toBe("string"); + expect(enumOf(v2Value, name)).toEqual( + expectedV2Members(name, enumOf(v1Value, name)), + ); + }); + + it("gives the scope sentinel to every field the cookbook says takes it", () => { + // The rule is "widest only on either axis", so the two fields that are + // widest + all in-scope on *both* axes are the only ones without it. Stated + // here as its own assertion because it is the whole point of decision 0023 + // and a silent omission would just read as a missing enum member. + for (const name of KEY_FEATURE_FIELDS) { + const value = entryValue(name); + const members = + value.enum ?? + value.items?.enum ?? + value.anyOf?.flatMap((b) => b.enum ?? []) ?? + []; + const takesIt = !WIDEST_ON_BOTH_AXES.has(name); + expect(members.includes(SCOPE_SENTINEL), `${name}`).toBe(takesIt); + } + }); + + it('every field offers an explicit "No information" value', () => { + // #858: an explicit "No information" terminates the resolver's fallback + // chain, as distinct from an absent entry. It only works if every field can + // express it — policyTypes is the one v1 field that could not. + for (const name of KEY_FEATURE_FIELDS) { + const value = entryValue(name); + // emissionsScope holds its enum in the common subschema it $refs, so follow + // the reference rather than skipping the field — skipping would let this + // test pass while the one $ref'd field quietly lost the value. + let options: string[]; + if (value.enum) { + options = value.enum; + } else if (value.items?.enum) { + options = value.items.enum; + } else if (value.anyOf) { + // emissionsScope composes the shared gas vocabulary with the scope + // sentinel, so "No information" comes from the branch that $refs it. + options = value.anyOf.flatMap((branch) => + branch.$ref === emissionsScope.$id + ? enumOf(emissionsScope, "emissionsScope.v1") + : (branch.enum ?? []), + ); + } else { + throw new Error(`${name}: could not resolve a value enum`); + } + expect(options, `${name} is missing "No information"`).toContain( + "No information", + ); + } + }); +}); + +describe("scopeSector.v2 tracks sector.v1", () => { + it("is sector.v1's display names plus the across sectors sentinel", () => { + const defs = sector.$defs; + if (!defs) throw new Error("sector.v1: expected $defs"); + const sectorNames = enumOf(defs.displayName, "sector.v1.displayName"); + const scopeNames = enumOf(scopeSector, "scopeSector.v2"); + expect(scopeNames).toContain("across sectors"); + expect( + [...scopeNames].filter((n) => n !== "across sectors").sort(), + ).toEqual([...sectorNames].sort()); + }); +}); + +/** + * #870's dataAvailability. The interesting properties are structural: that it is + * optional (authoring is incremental), that the row shape is closed, and that its + * vocabularies are `$ref`s rather than copies — the drift this file exists to + * catch is a vocabulary spelled out twice. + */ +describe("pathwayMetadata.v2 dataAvailability", () => { + const da = prop(v2, "dataAvailability", "v2"); + const byMetric = prop(da, "byMetric", "v2.dataAvailability"); + const row = items(byMetric, "v2.dataAvailability.byMetric"); + const rowProp = (name: string) => + prop(row, name, "v2.dataAvailability.byMetric.items"); + + it("is optional — a pathway without it is still valid", () => { + expect(v2.required ?? []).not.toContain("dataAvailability"); + }); + + it("requires both halves once present, and admits nothing else", () => { + expect([...(da.required ?? [])].sort()).toEqual(["byMetric", "overall"]); + expect(da.additionalProperties).toBe(false); + }); + + it("declares every row field as required, and admits nothing else", () => { + const expected = [ + "dataFormat", + "geography", + "granularity", + "metricName", + "scopeLimitations", + "sector", + "sectorSegment", + "timeResolution", + ]; + expect([...Object.keys(props(row, "row"))].sort()).toEqual(expected); + expect([...(row.required ?? [])].sort()).toEqual(expected); + expect(row.additionalProperties).toBe(false); + }); + + it("dedupes rows structurally as far as the schema can", () => { + // uniqueItems only catches wholly identical rows; two rows sharing a scope + // and differing elsewhere are caught by validateScopedEntries instead. + expect(byMetric.uniqueItems).toBe(true); + }); + + it("refs the shared vocabularies instead of restating them", () => { + const refs: Record = { + // NOT metric.v1: the availability row key is its own cookbook variable, + // which register item D16 settles as legitimately different from the + // pathway-level `metric`. Pinned here because pointing this back at + // metric.v1 is the regression that would silently re-narrow it to five + // Power metrics. + metricName: "common/dataAvailabilityMetric.v1.json#/$defs/displayName", + sector: "common/sector.v1.json#/$defs/displayName", + timeResolution: "common/dataAvailability.v1.json#/$defs/timeResolution", + dataFormat: "common/dataAvailability.v1.json#/$defs/dataFormat", + }; + for (const [name, suffix] of Object.entries(refs)) { + expect(rowProp(name).$ref).toBe( + `http://pathways.rmi.org/schema/${suffix}`, + ); + } + // sectorSegment is a list, so the $ref is on its items. + expect(items(rowProp("sectorSegment"), "sectorSegment").$ref).toBe( + "http://pathways.rmi.org/schema/common/sectorSegment.v1.json#/$defs/displayName", + ); + }); + + it.each(["geography", "granularity", "sectorSegment"])( + "makes %s a non-empty list", + (name) => { + // All three are Multiple in the cookbook, and real rows use that: the + // gold set has `Power generation; Energy storage` in one cell and a + // sixteen-token geography coverage cell. + const field = rowProp(name); + expect(field.type).toBe("array"); + expect(field.minItems).toBe(1); + expect(field.uniqueItems).toBe(true); + }, + ); + + it("scopes rows the same way keyFeatures does, so the same helpers apply", () => { + // A list here, a single token there -- the cookbook types Geography + // coverage as Multiple -- but drawn from the same vocabulary, which is what + // lets pathwayScopeOverlaps filter both. + expect(items(rowProp("geography"), "geography").$ref).toBe( + prop(entry("emissionsTrajectory"), "geography", "kf entry").$ref, + ); + }); + + it("retires the vocabularies decision 0021 dropped", () => { + // `geographyCoverage`'s Global/Regional/Country class restated, less + // precisely, what the geography token list already says. `access` went with + // `In tool`: dataFormat describes the source publication only, so there is + // no hosted-vs-publisher axis left for a paywall flag to qualify. + const defs = dataAvailability.$defs ?? {}; + expect(Object.keys(defs).sort()).toEqual([ + "dataFormat", + "granularityBreakdown", + "timeResolution", + ]); + for (const retired of ["geographyCoverage", "access"]) { + expect(Object.keys(props(row, "row"))).not.toContain(retired); + } + }); + + it("uses the plain sector enum, not scopeSector — there is no across sectors row", () => { + // Availability describes a concrete dataset, so the across sectors sentinel + // would have nothing to resolve to. + expect(rowProp("sector").$ref).not.toContain("scopeSector"); + expect(enumOf(scopeSector, "scopeSector.v2")).toContain("across sectors"); + }); + + it("draws granularity from technology.v1 and the breakdown vocabulary", () => { + // Two sources because the cookbook's granularity is conditional on + // (sector, metric): breakdown metrics list technologies, emissions metrics + // list a scope. Which applies to which is validateScopedEntries' job. + const granularity = rowProp("granularity"); + expect(granularity.uniqueItems).toBe(true); + const branches = items(granularity, "granularity").anyOf ?? []; + expect(branches.map((b) => b.$ref)).toEqual([ + "http://pathways.rmi.org/schema/common/technology.v1.json#/$defs/displayName", + "http://pathways.rmi.org/schema/common/dataAvailability.v1.json#/$defs/granularityBreakdown", + ]); + }); + + it.each(["geography", "granularity"])( + "makes %s a non-empty list rather than nullable", + (name) => { + // The cookbook replaces null with two distinct sentinels -- Unspecified + // (the pathway does not say) and Not covered (the pair is not covered) -- + // which a bare null cannot tell apart. + const field = rowProp(name); + expect(field.type).toBe("array"); + expect(field.minItems).toBe(1); + }, + ); + + it("caps the free-text fields", () => { + const limitations = rowProp("scopeLimitations"); + // Not nullable, for the same reason: "Unspecified" and "Not covered" are + // authored strings here. + expect(limitations.type).toBe("string"); + expect(limitations.minLength).toBe(1); + expect(limitations.maxLength).toBe(500); + // `overall` stays nullable: it summarises the pathway, not a metric, so it + // has no (sector, metric) pair to be uncovered for. + expect(prop(da, "overall", "v2.dataAvailability").type).toEqual([ + "string", + "null", + ]); + }); +}); diff --git a/src/types/common/dataAvailability.v1.d.ts b/src/types/common/dataAvailability.v1.d.ts new file mode 100644 index 00000000..5d968faa --- /dev/null +++ b/src/types/common/dataAvailability.v1.d.ts @@ -0,0 +1,58 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * How finely the underlying data is resolved over time. Where a pathway resolves a metric differently for different geographies, the most granular resolution available is the one recorded. + */ +export type TimeResolution = + | "2050 data point" + | "Medium-term data point" + | "10-year steps" + | "5-year steps" + | "1-year steps" + | "5/10-year steps" + | "Other time resolution" + | "Unspecified" + | "Not covered"; +/** + * In what form the source publication makes the metric's values available to a reader. Where values appear in more than one form the most extractable wins: Tabular before Text before Figure. + */ +export type DataFormat = "Tabular" | "Text" | "Figure" | "Not covered"; +/** + * Granularity values that are not technologies. A row's `granularity` is a flat array drawn from this vocabulary or from the sector's technologies, depending on the metric. + */ +export type GranularityBreakdown = + | "No information" + | "Unspecified" + | "Not covered" + | "Scope 1" + | "Scope 1 & 2" + | "Scope 1, 2 & 3" + | "Scope 1 & 3" + | "Total costs" + | "Capital costs, O&M, etc." + | "Other cost breakdown" + | "Total investment" + | "By sector" + | "By sector, part of value chain" + | "By technology" + | "By tech, part of value chain" + | "High-carbon assets lifetime assumed constant" + | "High-carbon assets lifetime face early retirement policy" + | "International connections only" + | "International and national transmission lines" + | "Transmission and distribution" + | "Scrap share as EAF input" + | "Scrap share as steel total input"; + +/** + * The closed vocabularies describing how a metric's underlying data can be obtained (#870). Values follow cookbook tpr_cookbook_20260924; see docs/cookbook/split/data_availability in RMI/tpr-tpc. + */ +export interface DataAvailabilityV1 { + timeResolution: TimeResolution; + dataFormat: DataFormat; + granularityBreakdown: GranularityBreakdown; +} diff --git a/src/types/common/dataAvailabilityMetric.v1.d.ts b/src/types/common/dataAvailabilityMetric.v1.d.ts new file mode 100644 index 00000000..cfe9ecfc --- /dev/null +++ b/src/types/common/dataAvailabilityMetric.v1.d.ts @@ -0,0 +1,43 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * Display name of the metric as the Data Availability table keys its rows. + */ +export type DisplayName = + | "Emissions intensity" + | "Absolute Emissions" + | "Capacity" + | "Generation" + | "Technology mix" + | "Transmission lines" + | "Technology cost" + | "Investment requirement" + | "Asset lifetime" + | "Emissions intensity (total)" + | "Emissions intensity (primary)" + | "Emissions intensity (secondary)" + | "Absolute emissions" + | "Technology mix (production by route)" + | "Steel production by technology (production route)" + | "Scrap share" + | "Emissions intensity (passenger)" + | "Emissions intensity (freight)" + | "Absolute emissions Well-to-Wheel (passenger)" + | "Absolute emissions Well-to-Wheel (freight)" + | "Total demand (passenger)" + | "Total demand (freight)" + | "Demand by propulsion technology (passenger)" + | "Demand by propulsion technology (freight)" + | "Demand share by propulsion technology (passenger)" + | "Demand share by propulsion technology (freight)"; + +/** + * The metric a dataAvailability row describes — the cookbook's 'Metrics – Extended' (#870). + */ +export interface DataAvailabilityMetricV1 { + displayName: DisplayName; +} diff --git a/src/types/common/scopeGeography.v2.d.ts b/src/types/common/scopeGeography.v2.d.ts new file mode 100644 index 00000000..daea07c2 --- /dev/null +++ b/src/types/common/scopeGeography.v2.d.ts @@ -0,0 +1,10 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeographyV2 = string; diff --git a/src/types/common/scopeSector.v2.d.ts b/src/types/common/scopeSector.v2.d.ts new file mode 100644 index 00000000..557b6779 --- /dev/null +++ b/src/types/common/scopeSector.v2.d.ts @@ -0,0 +1,26 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSectorV2 = + | "across sectors" + | "Land Use" + | "Agriculture" + | "Buildings" + | "Steel" + | "Cement" + | "Chemicals" + | "Coal Mining" + | "Oil (Upstream)" + | "Gas (Upstream)" + | "Power" + | "Automotive" + | "Aviation" + | "Rail" + | "Shipping" + | "Other"; diff --git a/src/types/common/sectorSegment.v1.d.ts b/src/types/common/sectorSegment.v1.d.ts new file mode 100644 index 00000000..a3debc0c --- /dev/null +++ b/src/types/common/sectorSegment.v1.d.ts @@ -0,0 +1,31 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * Display name of the sector segment as presented in tables. + */ +export type DisplayName = + | "No information" + | "Unspecified" + | "Not covered" + | "Upstream energy and fuels" + | "Passenger transport" + | "Freight transport" + | "Fuel extraction and processing" + | "Power generation" + | "Energy storage" + | "Transmission and distribution" + | "Mining" + | "Ironmaking" + | "Steelmaking" + | "Downstream"; + +/** + * Segments within a sector + */ +export interface SectorSegmentV1 { + displayName: DisplayName; +} diff --git a/src/types/common/technology.v1.d.ts b/src/types/common/technology.v1.d.ts index b1844249..bf6132a9 100644 --- a/src/types/common/technology.v1.d.ts +++ b/src/types/common/technology.v1.d.ts @@ -38,6 +38,26 @@ export type DisplayName = | "Active Mobility" | "Aviation Efficiency" | "Maritime Efficiency" + | "Geothermal" + | "Energy storage" + | "BOF" + | "BF-BOF" + | "BF-BOF+PCI" + | "BF-BOF+CCUS" + | "DRI-Melt-BOF" + | "EAF" + | "Scrap-EAF" + | "DRI-EAF" + | "DRI-EAF+H2" + | "DRI-EAF+CCS" + | "Electrolyser/Electrowinning" + | "Jet Fuel" + | "SAF" + | "Electricity" + | "Hydrogen" + | "HEFA" + | "PtL" + | "AtJ" | "Other"; /** diff --git a/src/types/index.ts b/src/types/index.ts index b4f56054..acd17b42 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,12 +1,23 @@ import type { FacetMode } from "../utils/searchUtils"; import type { PathwayMetadataV1 } from "./pathwayMetadata.v1"; +import type { PathwayMetadataV2 } from "./pathwayMetadata.v2"; import type { PublicationV1 } from "./common/publication.v1"; import type { GeographyV1 } from "./common/geography.v1"; -// Re-export the (current) versioned pathway metadata type as generic -export type PathwayMetadataType = PathwayMetadataV1; +// Re-export the (current) versioned pathway metadata type as generic. +// v2 as of #858: src/data/pathwayMetadata.ts validates against v2, so only v2 +// documents reach the app and this is the shape every consumer sees. +export type PathwayMetadataType = PathwayMetadataV2; export type PublicationType = PublicationV1; +// Both versions are exported for the migration window. v1 and v2 documents +// coexist in src/data — validateData routes each by its own $schema $id. +export type { PathwayMetadataV1, PathwayMetadataV2 }; + +/** A single scoped keyFeatures entry: {sector, geography, value} (#858). */ +export type ScopedKeyFeature = + PathwayMetadataV2["keyFeatures"][K][number]; + // Enum-like types derived from the schema export type PathwayType = PathwayMetadataType["pathwayType"]; export type Sector = PathwayMetadataType["sectors"][number]["name"]; diff --git a/src/types/pathwayMetadata.v1.d.ts b/src/types/pathwayMetadata.v1.d.ts index 7f36d7d1..733c4e0c 100644 --- a/src/types/pathwayMetadata.v1.d.ts +++ b/src/types/pathwayMetadata.v1.d.ts @@ -41,7 +41,7 @@ export interface PathwayMetadataV1 { description: string; publication: Publication; /** - * Type of the pathway pathway. + * Type of the pathway. */ pathwayType: "Normative" | "Exploratory" | "Predictive"; /** @@ -120,6 +120,26 @@ export interface PathwayMetadataV1 { | "Active Mobility" | "Aviation Efficiency" | "Maritime Efficiency" + | "Geothermal" + | "Energy storage" + | "BOF" + | "BF-BOF" + | "BF-BOF+PCI" + | "BF-BOF+CCUS" + | "DRI-Melt-BOF" + | "EAF" + | "Scrap-EAF" + | "DRI-EAF" + | "DRI-EAF+H2" + | "DRI-EAF+CCS" + | "Electrolyser/Electrowinning" + | "Jet Fuel" + | "SAF" + | "Electricity" + | "Hydrogen" + | "HEFA" + | "PtL" + | "AtJ" | "Other" )[]; }[]; diff --git a/src/types/pathwayMetadata.v2.d.ts b/src/types/pathwayMetadata.v2.d.ts new file mode 100644 index 00000000..4747c183 --- /dev/null +++ b/src/types/pathwayMetadata.v2.d.ts @@ -0,0 +1,676 @@ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * Name of the pathway. + */ +export type Label = import("./common/label.v1").LabelV1; +/** + * Bibliographic information about the report or dataset. + */ +export type Publication = import("./common/publication.v1").PublicationV1; +/** + * Geographical areas that the pathway covers. + */ +export type Geography = import("./common/geography.v1").GeographyV1; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector1 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography1 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector2 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography2 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector3 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography3 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector4 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography4 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector5 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography5 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector6 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography6 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * Defines which greenhouse gases are covered in the pathway's modeled emissions. + */ +export type EmissionsScope = + import("./common/emissionsScope.v1").EmissionsScopeV1; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector7 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography7 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector8 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography8 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector9 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography9 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The sector axis of a scoped keyFeatures entry: one of the sector display names, or the widest sentinel 'across sectors'. + */ +export type ScopeSector10 = import("./common/scopeSector.v2").ScopeSectorV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography10 = + import("./common/scopeGeography.v2").ScopeGeographyV2; +/** + * The geography axis of a scoped keyFeatures entry: 'Global', 'across regions', an ISO-3166-1 alpha-2 country code, or an author-defined region label. + */ +export type ScopeGeography11 = + import("./common/scopeGeography.v2").ScopeGeographyV2; + +/** + * A schema for the pathway metadata dataset in TPR. + */ +export interface PathwayMetadataV2 { + /** + * URI of the schema that validates this document (see https://json-schema.org/). + */ + $schema?: string; + /** + * The unique identifier for a pathway. + */ + id: string; + name: Label; + /** + * Brief description of the pathway. + */ + description: string; + publication: Publication; + /** + * Type of the pathway. + */ + pathwayType: "Normative" | "Exploratory" | "Predictive"; + /** + * Year by which net zero is reached in the pathway. If Pathway does not reach net zero, this field should be omitted. + */ + modelYearNetzero?: number; + /** + * Year from which the model starts. + */ + modelYearStart?: number; + /** + * Year in which the model ends. + */ + modelYearEnd?: number; + /** + * Modeled temperature increase expected by the pathway (in degrees Celsius). + */ + modelTempIncrease?: number; + geography: Geography; + /** + * Sectors that the pathway covers. + */ + sectors: { + /** + * Display name of a sector. + */ + name: + | "Land Use" + | "Agriculture" + | "Buildings" + | "Steel" + | "Cement" + | "Chemicals" + | "Coal Mining" + | "Oil (Upstream)" + | "Gas (Upstream)" + | "Power" + | "Automotive" + | "Aviation" + | "Rail" + | "Shipping" + | "Other"; + /** + * Technologies applicable to this sector. + * + * Items: Display name of the technology as presented in charts or tables. + */ + technologies: ( + | "Precision Agriculture" + | "Agroforestry" + | "Bioenergy Crops" + | "Energy Efficiency" + | "Smart Grids" + | "Renewable Heating" + | "Heat Pumps" + | "Building Automation" + | "Smart Appliances" + | "Insulation" + | "Carbon Capture and Storage" + | "Electrification" + | "Process Optimization" + | "Hydrogen Use" + | "Coal" + | "Oil" + | "Gas" + | "Wind" + | "Solar" + | "Nuclear" + | "Biomass" + | "Hydro" + | "Renewables" + | "Electric Vehicles" + | "Hydrogen Vehicles" + | "Biofuels" + | "Public Transport" + | "Active Mobility" + | "Aviation Efficiency" + | "Maritime Efficiency" + | "Geothermal" + | "Energy storage" + | "BOF" + | "BF-BOF" + | "BF-BOF+PCI" + | "BF-BOF+CCUS" + | "DRI-Melt-BOF" + | "EAF" + | "Scrap-EAF" + | "DRI-EAF" + | "DRI-EAF+H2" + | "DRI-EAF+CCS" + | "Electrolyser/Electrowinning" + | "Jet Fuel" + | "SAF" + | "Electricity" + | "Hydrogen" + | "HEFA" + | "PtL" + | "AtJ" + | "Other" + )[]; + /** + * Segments of this sector's value chain that the pathway covers -- it provides at least one relevant output metric for each. Members must be segments of this entry's `name`, enforced by scripts/schema-check-files.ts. + * + * Items: Display name of the sector segment as presented in tables. + */ + segments?: import("./common/sectorSegment.v1").SectorSegmentV1["displayName"][]; + }[]; + /** + * Narrative description of the pathway. Replaces v1's expertOverview: in the v1 corpus this is the '#### Pathway Description' section of it. v1's separate pathwayOverview field is retired without replacement, not merged in here -- the two texts restate each other, so merging them read as immediate self-repetition. null means no description is available. + */ + pathwayDescription: string | null; + /** + * Items: Display name of the metric + */ + metric: import("./common/metric.v1").MetricV1["displayName"][]; + /** + * Key features of the pathway. Every field is an array of {sector, geography, value} entries (#858), so a pathway can hold different values for different parts of its coverage. A non-varying feature carries exactly one entry at the widest applicable scope: sector 'across sectors' for a multi-sector pathway else its lone sector, and geography 'Global' else the pathway's widest declared region or country. An entry that is absent at some scope means the resolver keeps broadening until it finds one; an explicit "No information" value is a real authored value that terminates that fallback chain and displays at its own scope. An empty array means nothing is authored at any scope. + */ + keyFeatures: { + /** + * Describes the overall trend of greenhouse gas emissions over time, from continued growth to rapid decline. Scoped: see keyFeatures. + */ + emissionsTrajectory: { + sector: ScopeSector; + geography: ScopeGeography; + value: + | "No information" + | "Significant increase" + | "Moderate increase" + | "Minor increase" + | "Low or no change" + | "Minor decrease" + | "Moderate decrease" + | "Significant decrease"; + }[]; + /** + * Indicates how efficiently energy is used to produce economic output across the sectors covered in the pathway. Scoped: see keyFeatures. + */ + energyEfficiency: { + sector: ScopeSector1; + geography: ScopeGeography1; + value: + | "No information" + | "Significant deterioration" + | "Moderate deterioration" + | "Minor deterioration" + | "Low or no change" + | "Minor improvement" + | "Moderate improvement" + | "Significant improvement" + | "Not applicable at this scope level"; + }[]; + /** + * Captures the change in total energy consumption, driven by factors such as socio-economic development, technology shifts and consumer behavior. Scoped: see keyFeatures. + */ + energyDemand: { + sector: ScopeSector2; + geography: ScopeGeography2; + value: + | "No information" + | "Significant decrease" + | "Moderate decrease" + | "Minor decrease" + | "Low or no change" + | "Minor increase" + | "Moderate increase" + | "Significant increase"; + }[]; + /** + * Represents the extent to which energy end-uses transition from fossil fuels to electricity. Scoped: see keyFeatures. + */ + electrification: { + sector: ScopeSector3; + geography: ScopeGeography3; + value: + | "No information" + | "Significant decrease" + | "Moderate decrease" + | "Minor decrease" + | "Low or no change" + | "Minor increase" + | "Moderate increase" + | "Significant increase" + | "Not Applicable" + | "Not applicable at this scope level"; + }[]; + /** + * Identifies the types of policies modeled as drivers of the pathway, such as carbon pricing, subsidies, or mandated phaseouts of specific technologies. Scoped: see keyFeatures. + */ + policyTypes: { + sector: ScopeSector4; + geography: ScopeGeography4; + /** + * @minItems 1 + */ + value: [ + ( + | "No information" + | "Carbon price" + | "Feed-in tariffs" + | "Performance standards" + | "Phaseout dates" + | "Subsidies" + | "Target technology shares" + | "Other" + | "None" + | "Not applicable at this scope level" + ), + ...( + | "No information" + | "Carbon price" + | "Feed-in tariffs" + | "Performance standards" + | "Phaseout dates" + | "Subsidies" + | "Target technology shares" + | "Other" + | "None" + | "Not applicable at this scope level" + )[], + ]; + }[]; + /** + * Describes how technology costs evolve over time, from static cost assumptions to rapidly declining costs (e.g., via learning curves). Scoped: see keyFeatures. + */ + technologyCostTrend: { + sector: ScopeSector5; + geography: ScopeGeography5; + value: + | "No information" + | "Increase" + | "Low or no change" + | "Decrease" + | "Not applicable at this scope level"; + }[]; + /** + * Defines which greenhouse gases are covered in the pathway's modeled emissions. Scoped: see keyFeatures. + */ + emissionsScope: { + sector: ScopeSector6; + geography: ScopeGeography6; + value: EmissionsScope | "Not applicable at this scope level"; + }[]; + /** + * Represents the overall stringency and intent of modeled policies relative to climate targets, often reflecting if and how far the included policies go beyond currently legislated ones. Scoped: see keyFeatures. + */ + policyAmbition: { + sector: ScopeSector7; + geography: ScopeGeography7; + value: + | "No information" + | "No policies included" + | "Current/legislated policies" + | "Current and drafted policies" + | "NDCs, unconditional only" + | "NDCs incl. conditional targets" + | "High ambition policies" + | "Other policy ambition" + | "Not applicable at this scope level"; + }[]; + /** + * Specifies the level of granularity in cost data, such as total system costs or detailed CAPEX/OPEX breakdowns. Scoped: see keyFeatures. + */ + technologyCostsDetail: { + sector: ScopeSector8; + geography: ScopeGeography8; + value: + | "No information" + | "Total costs" + | "Capital costs, O&M, etc." + | "Other cost breakdown" + | "Not applicable at this scope level"; + }[]; + /** + * Lists emerging or breakthrough technologies that are explicitly modeled within the pathway. These are considered in technology deployment too. Scoped: see keyFeatures. + */ + newTechnologiesIncluded: { + sector: ScopeSector9; + geography: ScopeGeography9; + /** + * @minItems 1 + */ + value: [ + ( + | "No information" + | "No new technologies" + | "CCUS" + | "DAC" + | "Green H2/ammonia" + | "SAF" + | "Battery storage" + | "EGS/AGS" + | "SMR" + | "Other new technologies" + | "Not applicable at this scope level" + ), + ...( + | "No information" + | "No new technologies" + | "CCUS" + | "DAC" + | "Green H2/ammonia" + | "SAF" + | "Battery storage" + | "EGS/AGS" + | "SMR" + | "Other new technologies" + | "Not applicable at this scope level" + )[], + ]; + }[]; + /** + * Summarizes how investment requirements are quantified, from total system to sector-level or supply-chain detail. Scoped: see keyFeatures. + */ + investmentNeeds: { + sector: ScopeSector10; + geography: ScopeGeography10; + value: + | "No information" + | "Total investment" + | "By sector" + | "By sector, part of value chain" + | "By technology" + | "By tech, part of value chain" + | "Not applicable at this scope level"; + }[]; + }; + /** + * The drivers that shape this pathway's outcomes. Every field is required but nullable: null means the driver is not a core driver for this pathway, as distinct from a driver that is present but undescribed. + */ + coreDrivers: { + /** + * Policies modeled as a driver of this pathway. + */ + policies: string | null; + /** + * Emissions targets or constraints driving this pathway. + */ + emissionsTargets: string | null; + /** + * Technology cost assumptions driving this pathway. + */ + technologyCosts: string | null; + /** + * Changes in investment driving this pathway. + */ + investmentChange: string | null; + /** + * Macroeconomic assumptions driving this pathway. + */ + macroeconomicDrivers: string | null; + /** + * Behavioral or demand-side shifts driving this pathway. + */ + behavioralShifts: string | null; + /** + * Any other core driver of this pathway. + */ + otherDrivers: string | null; + }; + /** + * Conditions the pathway's outcomes depend on. Descriptive only -- deliberately NOT part of the #869 inheritance chain, so these are not scoped by geography. + */ + dependencies: { + /** + * Category of the dependency. + */ + dependency_name: + | "Policy strategy" + | "Regulatory framework" + | "Market and economics" + | "Public acceptance" + | "Consumer and client behavior" + | "Infrastructure and logistics" + | "Technology" + | "Resource availability" + | "Environmental impacts and ecosystem services" + | "Labor availability"; + /** + * What the pathway depends on, in prose. + */ + dependency_description: string; + /** + * Display name of a sector. + */ + sector: + | "Land Use" + | "Agriculture" + | "Buildings" + | "Steel" + | "Cement" + | "Chemicals" + | "Coal Mining" + | "Oil (Upstream)" + | "Gas (Upstream)" + | "Power" + | "Automotive" + | "Aviation" + | "Rail" + | "Shipping" + | "Other"; + /** + * Strength of the evidence for this dependency. + */ + evidence_type: "Quantitative" | "Qualitative" | "Anecdotal" | "No evidence"; + }[]; + /** + * Where and how the data behind each metric can be obtained (#870). Optional: authoring is incremental, and a pathway with no entry yet is not an invalid pathway. Absent means unknown, NOT unavailable -- a row's `Not covered` values say unavailable. + */ + dataAvailability?: { + /** + * Summary of the full timeseries file we host, plus anything that does not fit the per-metric rows. Null where there is nothing to add. + */ + overall: string | null; + /** + * One row per (metricName, sector, sectorSegment, geography set). Each combination may appear only once -- enforced by scripts/schema-check-files.ts, because uniqueItems compares whole entries and so permits two rows that agree on the scope and disagree on everything else. The cookbook expects a row for every allowable (sector, metric) pair in a covered sector, with an uncovered pair recorded as 'Not covered' rather than omitted. + */ + byMetric: { + /** + * Metric this row describes -- the cookbook's 'Metrics - Extended', which is a different vocabulary from the pathway-level `metric` field (register item D16). Must be a metric of this row's `sector`, enforced by scripts/schema-check-files.ts. + */ + metricName: import("./common/dataAvailabilityMetric.v1").DataAvailabilityMetricV1["displayName"]; + /** + * Display name of a sector. + */ + sector: import("./common/sector.v1").SectorV1["displayName"]; + /** + * Segments of this row's sector that the metric covers. `Unspecified` where the pathway names none, `Not covered` where the sector-metric pair is not covered; either must then be the only member. Members must be segments of this row's `sector`, enforced by scripts/schema-check-files.ts. + * + * @minItems 1 + * + * Items: Display name of the sector segment as presented in tables. + */ + sectorSegment: [ + import("./common/sectorSegment.v1").SectorSegmentV1["displayName"], + ...import("./common/sectorSegment.v1").SectorSegmentV1["displayName"][], + ]; + /** + * Geographies this row covers, as the cookbook's `Geography coverage`: a subset of the pathway's own declared geography, enforced by scripts/schema-check-files.ts. Same scope tokens as keyFeatures, so the table can be filtered by the detail page's geography selection (#872) using the existing scope helpers. `Unspecified` where the pathway does not say which, `Not covered` where the sector-metric pair is not covered; either must then be the only member. + * + * @minItems 1 + */ + geography: [ScopeGeography11, ...ScopeGeography11[]]; + /** + * How finely the underlying data is resolved over time. + */ + timeResolution: + | "2050 data point" + | "Medium-term data point" + | "10-year steps" + | "5-year steps" + | "1-year steps" + | "5/10-year steps" + | "Other time resolution" + | "Unspecified" + | "Not covered"; + /** + * In what form the source publication reports this metric's values. + */ + dataFormat: "Tabular" | "Text" | "Figure" | "Not covered"; + /** + * Dimensions the metric is broken down by. Members are either technologies of this row's sector -- enforced by scripts/schema-check-files.ts, the same rule as sectors[].technologies -- or values from the granularityBreakdown vocabulary. `Unspecified` where the pathway does not clarify the breakdown, `Not covered` where the sector-metric pair is not covered; either must then be the only member. + * + * @minItems 1 + */ + granularity: [ + ( + | import("./common/technology.v1").TechnologyV1["displayName"] + | ( + | "No information" + | "Unspecified" + | "Not covered" + | "Scope 1" + | "Scope 1 & 2" + | "Scope 1, 2 & 3" + | "Scope 1 & 3" + | "Total costs" + | "Capital costs, O&M, etc." + | "Other cost breakdown" + | "Total investment" + | "By sector" + | "By sector, part of value chain" + | "By technology" + | "By tech, part of value chain" + | "High-carbon assets lifetime assumed constant" + | "High-carbon assets lifetime face early retirement policy" + | "International connections only" + | "International and national transmission lines" + | "Transmission and distribution" + | "Scrap share as EAF input" + | "Scrap share as steel total input" + ) + ), + ...( + | import("./common/technology.v1").TechnologyV1["displayName"] + | ( + | "No information" + | "Unspecified" + | "Not covered" + | "Scope 1" + | "Scope 1 & 2" + | "Scope 1, 2 & 3" + | "Scope 1 & 3" + | "Total costs" + | "Capital costs, O&M, etc." + | "Other cost breakdown" + | "Total investment" + | "By sector" + | "By sector, part of value chain" + | "By technology" + | "By tech, part of value chain" + | "High-carbon assets lifetime assumed constant" + | "High-carbon assets lifetime face early retirement policy" + | "International connections only" + | "International and national transmission lines" + | "Transmission and distribution" + | "Scrap share as EAF input" + | "Scrap share as steel total input" + ) + )[], + ]; + /** + * Caveats on what the data does and does not cover, in prose: the boundaries of the sector model, what is in and out of scope, and for emissions metrics the emissions scope. `Unspecified` where the pathway does not say, `Not covered` where the sector-metric pair is not covered. + */ + scopeLimitations: string; + }[]; + }; +} diff --git a/src/utils/keyFeatureScope.test.ts b/src/utils/keyFeatureScope.test.ts new file mode 100644 index 00000000..3c2415ee --- /dev/null +++ b/src/utils/keyFeatureScope.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect } from "vitest"; +import { + entryValues, + sectorScopeContains, + entryISOSet, + geographyScopeOverlaps, + entriesInScope, + valuesInScope, + widestValue, +} from "./keyFeatureScope"; +import { ABSENT_FILTER_TOKEN } from "./absent"; +import type { PathwayMetadataType } from "../types"; + +/** + * A pathway covering Power and Steel, with one mapped region, one unmapped region + * (the NGFS shape pending #801), and one standalone country. + */ +function pathway(over: Partial = {}): PathwayMetadataType { + return { + sectors: [ + { name: "Power", technologies: [] }, + { name: "Steel", technologies: [] }, + ], + geography: { + regions: { + "South East Asia": ["ID", "TH", "VN"], + "Unmapped Region": [], + }, + country: ["US"], + }, + ...over, + } as unknown as PathwayMetadataType; +} + +const e = (sector: string, geography: string, value: string | string[]) => ({ + sector, + geography, + value, +}); + +describe("entryValues", () => { + it("flattens scalar and array-valued entries alike", () => { + expect( + entryValues([ + e("Power", "Global", "A"), + e("Steel", "Global", ["B", "C"]), + ]), + ).toEqual(["A", "B", "C"]); + }); + + it("returns [] for an empty entry list", () => { + expect(entryValues([])).toEqual([]); + }); + + it("returns [] for a non-array — a stale v1 scalar degrades, not throws", () => { + // Relevant while v1 and v2 coexist: a v1-shaped scalar must not blow up here. + expect(entryValues("Significant decrease")).toEqual([]); + expect(entryValues(null)).toEqual([]); + expect(entryValues(undefined)).toEqual([]); + }); +}); + +describe("sectorScopeContains", () => { + const declared = ["Power", "Steel"]; + + it("matches an exact sector", () => { + expect(sectorScopeContains("Power", "Power", declared)).toBe(true); + }); + + it("does not match a different sector", () => { + expect(sectorScopeContains("Power", "Steel", declared)).toBe(false); + }); + + it("across sectors contains any sector the pathway declares", () => { + expect(sectorScopeContains("across sectors", "Steel", declared)).toBe(true); + }); + + it("across sectors is NOT a universal match", () => { + // Per Jacob on #869: across sectors is the union of the pathway's own sectors, + // so a pathway covering only Power and Steel does not answer a Cement query. + expect(sectorScopeContains("across sectors", "Cement", declared)).toBe( + false, + ); + }); +}); + +describe("entryISOSet", () => { + it("returns null for Global, meaning everything", () => { + expect(entryISOSet("Global", pathway())).toBeNull(); + }); + + it("resolves a region label through the pathway's own mapping", () => { + expect( + [...(entryISOSet("South East Asia", pathway()) ?? [])].sort(), + ).toEqual(["ID", "TH", "VN"]); + }); + + it("resolves a bare country code", () => { + expect([...(entryISOSet("US", pathway()) ?? [])]).toEqual(["US"]); + }); + + it("resolves an unmapped region to the empty set, not to everything", () => { + // #801: a region the publication never mapped must match nothing rather than + // silently behaving like Global. + expect(entryISOSet("Unmapped Region", pathway())?.size).toBe(0); + }); + + it("resolves across regions to the pathway's whole ISO coverage", () => { + const set = entryISOSet("across regions", pathway()); + expect([...(set ?? [])].sort()).toEqual(["ID", "TH", "US", "VN"]); + }); + + it("resolves an unrecognised label to the empty set", () => { + expect(entryISOSet("Souteast Asia", pathway())?.size).toBe(0); + }); +}); + +describe("geographyScopeOverlaps", () => { + const p = pathway(); + + it("a Global entry answers any query", () => { + expect(geographyScopeOverlaps("Global", "TH", p)).toBe(true); + }); + + it("a region entry answers a country inside it", () => { + expect(geographyScopeOverlaps("South East Asia", "TH", p)).toBe(true); + }); + + it("a country entry answers a region query that includes it", () => { + // Overlap, not containment. "Southeast Asia" is the query vocabulary's + // label (11 ISO codes) and includes TH, so a Thailand-scoped entry is a + // match. Under the old containment rule this was false, which is what made + // region filters drop pathways whose own region list differs by a country. + expect(geographyScopeOverlaps("TH", "Southeast Asia", p)).toBe(true); + }); + + it("tolerates a publication label the query vocabulary does not share", () => { + // The pathway's own regions are named "South East Asia" (with spaces); the + // filter vocabulary calls it "Southeast Asia". The two lists also differ by + // one country (TL), so containment would fail even once spelling matched. + // Overlap makes the intersection sufficient, which is the point of #783. + expect(geographyScopeOverlaps("South East Asia", "Southeast Asia", p)).toBe( + true, + ); + }); + + it("does not match a country outside the entry's region", () => { + expect(geographyScopeOverlaps("South East Asia", "US", p)).toBe(false); + }); + + it("only a Global entry answers a Global query", () => { + // Mirrors the geography facet's `wantGlobal && pGlobal`: selecting "Global" + // means whole-world pathways, not "match anything". + expect(geographyScopeOverlaps("Global", "Global", p)).toBe(true); + expect(geographyScopeOverlaps("South East Asia", "Global", p)).toBe(false); + }); + + it("an unrecognised token matches nothing", () => { + // "South East Asia" is a publication label, not a query-vocabulary token, so + // it resolves to an empty ISO set and must not vacuously match. + expect(geographyScopeOverlaps("TH", "South East Asia", p)).toBe(false); + expect(geographyScopeOverlaps("TH", "Souteast Asia", p)).toBe(false); + // Including for a Global entry, which answers any *real* selection. The + // Global shortcut used to run first and answered a stale token too. + expect(geographyScopeOverlaps("Global", "Souteast Asia", p)).toBe(false); + }); + + it("the absent bucket does not constrain which scope to read", () => { + expect( + geographyScopeOverlaps("South East Asia", ABSENT_FILTER_TOKEN, p), + ).toBe(true); + }); + + it("an unmapped region overlaps nothing", () => { + expect(geographyScopeOverlaps("Unmapped Region", "TH", p)).toBe(false); + }); +}); + +describe("entriesInScope", () => { + const entries = [ + e("across sectors", "Global", "wide"), + e("Power", "South East Asia", "power-sea"), + e("Steel", "US", "steel-us"), + ]; + const p = pathway(); + + it("returns every entry when neither axis is filtered", () => { + expect(entriesInScope(entries, {}, p)).toHaveLength(3); + }); + + it("keeps entries whose sector contains the queried sector", () => { + expect( + entriesInScope(entries, { sectors: ["Power"] }, p).map((x) => x.value), + ).toEqual(["wide", "power-sea"]); + }); + + it("keeps entries whose geography contains the queried country", () => { + expect( + entriesInScope(entries, { geographies: ["TH"] }, p).map((x) => x.value), + ).toEqual(["wide", "power-sea"]); + }); + + it("applies both axes together", () => { + expect( + entriesInScope( + entries, + { sectors: ["Steel"], geographies: ["US"] }, + p, + ).map((x) => x.value), + ).toEqual(["wide", "steel-us"]); + }); + + it("excludes an entry when only one axis matches", () => { + // Steel data exists, but only for the US — not for Thailand. + expect( + entriesInScope( + entries, + { sectors: ["Steel"], geographies: ["TH"] }, + p, + ).map((x) => x.value), + ).toEqual(["wide"]); + }); + + it("treats several selections on one axis as 'any of them'", () => { + expect( + entriesInScope(entries, { sectors: ["Power", "Steel"] }, p).map( + (x) => x.value, + ), + ).toEqual(["wide", "power-sea", "steel-us"]); + }); + + it("can narrow to nothing when no entry covers the query", () => { + const narrow = [e("Power", "South East Asia", "power-sea")]; + expect(entriesInScope(narrow, { geographies: ["US"] }, p)).toEqual([]); + }); + + it("returns [] for absent entries", () => { + expect(entriesInScope(undefined, { sectors: ["Power"] }, p)).toEqual([]); + expect(entriesInScope([], { sectors: ["Power"] }, p)).toEqual([]); + }); +}); + +describe("entriesInScope — the ABSENT/None token", () => { + // Regression: the None bucket is a predicate about the pathway ("has no + // sectors"), not a scope. geographyScopeContains always ignored it, but the + // sector axis compared it as a sector name, so Sector=None plus a keyFeature + // facet filtered out every entry and the pathway looked empty. + const entries = [ + e("across sectors", "Global", "wide"), + e("Power", "South East Asia", "power-sea"), + ]; + const p = pathway(); + + it("does not constrain the sector axis", () => { + expect( + entriesInScope(entries, { sectors: [ABSENT_FILTER_TOKEN] }, p).map( + (x) => x.value, + ), + ).toEqual(["wide", "power-sea"]); + }); + + it("does not constrain the geography axis", () => { + expect( + entriesInScope(entries, { geographies: [ABSENT_FILTER_TOKEN] }, p).map( + (x) => x.value, + ), + ).toEqual(["wide", "power-sea"]); + }); + + it("is ignored on both axes at once", () => { + expect( + entriesInScope( + entries, + { sectors: [ABSENT_FILTER_TOKEN], geographies: [ABSENT_FILTER_TOKEN] }, + p, + ), + ).toHaveLength(2); + }); + + it("still applies the concrete tokens alongside it", () => { + // None + Power: the real sector still narrows; None adds no constraint. + expect( + entriesInScope( + entries, + { sectors: [ABSENT_FILTER_TOKEN, "Steel"] }, + p, + ).map((x) => x.value), + ).toEqual(["wide"]); + }); + + it("treats it the same for a pathway that declares no sectors at all", () => { + const noSectors = pathway({ sectors: [] }); + expect( + entriesInScope( + [e("across sectors", "Global", "wide")], + { sectors: [ABSENT_FILTER_TOKEN] }, + noSectors, + ), + ).toHaveLength(1); + }); +}); + +describe("widestValue", () => { + it("returns undefined when nothing is authored", () => { + expect(widestValue([])).toBeUndefined(); + expect(widestValue(undefined)).toBeUndefined(); + }); + + it("returns the only value when there is one entry", () => { + expect(widestValue([e("across sectors", "Global", "only")])).toBe("only"); + }); + + it("prefers across sectors over a named sector", () => { + expect( + widestValue([ + e("Power", "Global", "narrow"), + e("across sectors", "Global", "wide"), + ]), + ).toBe("wide"); + }); + + it("prefers Global over a region, and a region over a country", () => { + expect( + widestValue([ + e("across sectors", "TH", "country"), + e("across sectors", "South East Asia", "region"), + e("across sectors", "Global", "global"), + ]), + ).toBe("global"); + expect( + widestValue([ + e("across sectors", "TH", "country"), + e("across sectors", "South East Asia", "region"), + ]), + ).toBe("region"); + }); + + it("ranks sector ahead of geography", () => { + // A across sectors entry wins even when its geography is narrower, matching the + // "sector > geography" precedence #869 defines for its cost model. + expect( + widestValue([ + e("Power", "Global", "power-global"), + e("across sectors", "TH", "cross-th"), + ]), + ).toBe("cross-th"); + }); + + it("preserves an array value intact", () => { + expect(widestValue([e("across sectors", "Global", ["a", "b"])])).toEqual([ + "a", + "b", + ]); + }); + + it("degrades to undefined for a stale v1 scalar", () => { + expect(widestValue("Moderate decrease")).toBeUndefined(); + }); +}); + +describe("valuesInScope", () => { + it("flattens the in-scope entries' values", () => { + const entries = [ + e("across sectors", "Global", ["a", "b"]), + e("Steel", "US", "c"), + ]; + expect(valuesInScope(entries, { sectors: ["Power"] }, pathway())).toEqual([ + "a", + "b", + ]); + }); +}); diff --git a/src/utils/keyFeatureScope.ts b/src/utils/keyFeatureScope.ts new file mode 100644 index 00000000..04e644f2 --- /dev/null +++ b/src/utils/keyFeatureScope.ts @@ -0,0 +1,248 @@ +/** + * Reading v2's scoped keyFeatures entries (#858). + * + * Each keyFeature is an array of `{sector, geography, value}` entries, so + * "what is this pathway's emissionsTrajectory?" now depends on which part of the + * pathway's coverage you are asking about. This module answers two questions the + * search layer needs: + * + * - which entries are relevant to the user's current sector/geography filter + * ({@link entriesInScope}), and + * - what values those entries carry ({@link entryValues}). + * + * Deliberately **inclusion only**: an entry is relevant when its scope meets the + * query on both axes — sector by containment, geography by non-empty ISO + * intersection (see geographyScopeOverlaps for why the two axes differ). There is + * no cost model, no ranking, and no notion of how far the query had to broaden — + * that is #869's resolver, which is where strict containment belongs, as the basis + * for ranking rather than as a hard filter. + */ +import type { GeographyCode, PathwayMetadataType } from "../types"; +import { pathwayISOCoverage, toISO2 } from "./geographyUtils"; +import { selectedGeographyToISO } from "./filterRegions"; +import { ABSENT_FILTER_TOKEN } from "./absent"; + +/** Sector sentinel: the union of *this pathway's own* declared sectors. */ +export const ACROSS_SECTORS = "across sectors"; +/** Geography sentinels: everything, and a multi-region non-global aggregate. */ +export const GLOBAL_SCOPE = "Global"; +export const ACROSS_REGIONS = "across regions"; + +/** One scoped entry, structurally — the 11 fields differ only in `value`. */ +export interface ScopedEntry { + sector: string; + geography: string; + value: string | string[]; +} + +/** + * Coerce a field's value to entries, tolerating anything that is not an array. + * + * Takes `unknown` on purpose. While v1 and v2 coexist a v1-shaped scalar can + * reach these helpers from a hand-built fixture or a stale mock, and degrading to + * "no entries" is better than throwing. `Array.isArray` narrows a typed + * `readonly T[]` union to `any[]`, so the cast is what keeps this type-safe. + */ +function asEntries(value: unknown): readonly ScopedEntry[] { + return Array.isArray(value) ? (value as readonly ScopedEntry[]) : []; +} + +/** Flatten one field's entries to the values they carry, array-valued or not. */ +export function entryValues(entries: unknown): string[] { + return asEntries(entries).flatMap((e) => { + if (Array.isArray(e.value)) return e.value; + return e.value != null ? [e.value] : []; + }); +} + +/** + * Does an entry's sector scope contain a queried sector? + * + * `across sectors` is **not** a universal match (per Jacob on #869): it means the + * union of the sectors this pathway declares, so a pathway covering only Y and Z + * does not answer a query for sector X even though its `across sectors` values are + * nominally broad enough. + */ +export function sectorScopeContains( + entrySector: string, + querySector: string, + declaredSectors: readonly string[], +): boolean { + if (entrySector === querySector) return true; + if (entrySector === ACROSS_SECTORS) + return declaredSectors.includes(querySector); + return false; +} + +/** + * The ISO codes an entry's geography scope covers, or `null` for "everything". + * + * A region label resolves through the pathway's own `geography.regions` mapping, + * which is why an unmapped region (empty member array, e.g. the NGFS files + * pending #801) resolves to the empty set and therefore matches nothing rather + * than matching everything. + */ +export function entryISOSet( + entryGeography: string, + pathway: PathwayMetadataType, +): Set | null { + if (entryGeography === GLOBAL_SCOPE) return null; + + const geo = pathway.geography; + if (entryGeography === ACROSS_REGIONS) return pathwayISOCoverage(geo); + + const members = geo?.regions?.[entryGeography]; + if (Array.isArray(members)) return new Set(members); + + const iso = toISO2(entryGeography); + if (iso) return new Set([iso as GeographyCode]); + + return new Set(); +} + +/** + * Does an entry's geography scope overlap a selected geography token? + * + * **Overlap, not containment** — a non-empty intersection is a match. Requiring + * the query to be a strict subset of the entry looks tidier but is wrong here, + * because the query vocabulary and each publication's own region membership are + * different lists by design (#783), so they rarely coincide exactly. Concretely: + * the filter vocabulary's "Southeast Asia" carries 11 codes including TL, while + * ACE's own "South East Asia" carries 10 and omits it. Under containment that one + * country made every ACE pathway vanish whenever a region filter was combined + * with a keyFeature facet, even though the geography facet itself kept them -- + * it has always used overlap (`isoSets.some(overlaps)` in filterPathways). This + * now matches that behaviour, so the two cannot disagree about the same pathway. + * + * "Global" stays a distinct predicate rather than "every code", again mirroring + * the facet (`wantGlobal && pGlobal`): a Global query is answered only by a + * Global-scoped entry, so selecting it does not quietly match everything. + */ +export function geographyScopeOverlaps( + entryGeography: string, + queryToken: string, + pathway: PathwayMetadataType, +): boolean { + const query = selectedGeographyToISO(queryToken); + // The "None" bucket says nothing about which scope to read. entriesInScope now + // strips it before calling this, so the guard is only for direct callers. + if (query.kind === "absent") return true; + + // An unrecognised token resolves to an empty ISO set. Checked before the + // Global shortcut below, which would otherwise answer it: a stale selection + // must match nothing, including a Global-scoped entry. + if (query.kind === "iso" && query.iso.size === 0) return false; + + const entrySet = entryISOSet(entryGeography, pathway); + if (entrySet === null) return true; // a Global entry answers any real selection + + if (query.kind === "global") return false; // only a Global entry answers Global + // An unrecognised token, or a region the publication never mapped, yields an + // empty set and so overlaps nothing -- a stale selection matches nothing + // rather than everything. + for (const code of query.iso) if (entrySet.has(code)) return true; + return false; +} + +export interface ScopeQuery { + /** Selected sector tokens; empty leaves the sector axis unconstrained. */ + sectors?: readonly string[]; + /** Selected geography tokens; empty leaves the geography axis unconstrained. */ + geographies?: readonly string[]; +} + +/** Selected tokens with the ABSENT/"None" bucket removed — see entriesInScope. */ +function dropAbsent(tokens: readonly string[] | undefined): readonly string[] { + if (!tokens || tokens.length === 0) return []; + return tokens.filter((t) => t !== ABSENT_FILTER_TOKEN); +} + +/** + * The entries relevant to the user's current filter. + * + * With neither axis filtered this returns every entry, which is what makes the + * blank-search view behave exactly as it did on v1 data. Multiple selections on + * an axis are treated as "contains any of them": the ANY/ALL facet mode the user + * picked governs *value* matching, not which scope to read from, so a + * sector=[Power, Steel] selection makes entries for either sector relevant. + */ +export function entriesInScope( + entries: unknown, + query: ScopeQuery, + pathway: PathwayMetadataType, +): ScopedEntry[] { + const all = asEntries(entries); + // The "None" bucket is a predicate about the *pathway* ("has no sectors" / + // "has no geography"), not a scope to read values from, so it must not + // constrain either axis. Dropping it here rather than in each axis helper keeps + // the two consistent: previously geographyScopeOverlaps ignored the token but + // sectorScopeContains compared it as if it were a sector name, so selecting + // Sector=None alongside a keyFeature facet filtered out every entry and made + // the pathway look like it held no values at all. + const sectors = dropAbsent(query.sectors); + const geographies = dropAbsent(query.geographies); + if (sectors.length === 0 && geographies.length === 0) return [...all]; + + const declared = (pathway.sectors ?? []).map((s) => s.name); + + return all.filter((entry) => { + const sectorOk = + sectors.length === 0 || + sectors.some((s) => sectorScopeContains(entry.sector, s, declared)); + if (!sectorOk) return false; + const geoOk = + geographies.length === 0 || + geographies.some((g) => + geographyScopeOverlaps(entry.geography, g, pathway), + ); + return geoOk; + }); +} + +/** Values of the entries relevant to the query — what a facet matches against. */ +export function valuesInScope( + entries: unknown, + query: ScopeQuery, + pathway: PathwayMetadataType, +): string[] { + return entryValues(entriesInScope(entries, query, pathway)); +} + +/** + * How broad a scope axis is, lower being broader. Derived from the token alone so + * this works without pathway context, which is what {@link widestValue}'s callers + * (the rendering components) have available. + */ +function sectorBreadth(sector: string): number { + return sector === ACROSS_SECTORS ? 0 : 1; +} + +function geographyBreadth(geography: string): number { + if (geography === GLOBAL_SCOPE) return 0; + if (geography === ACROSS_REGIONS) return 1; + // A two-letter token is a country code; anything longer is a region label. + return /^[A-Za-z]{2}$/.test(geography) ? 3 : 2; +} + +/** + * The value at the broadest scope a field declares. + * + * **Provisional.** This is a placeholder for #869's resolver, which will pick the + * value for the scope the *user* is looking at and report how far it had to + * broaden so #859 can badge it. Until then the components show the widest value, + * which is what v1 effectively showed — every codemod-migrated pathway has exactly + * one entry, at its widest scope — so rendering is unchanged for current data. + * + * Returns `undefined` when nothing is authored at any scope, which the callers + * already render as "No information". + */ +export function widestValue(entries: unknown): string | string[] | undefined { + const all = asEntries(entries); + if (all.length === 0) return undefined; + const ranked = [...all].sort( + (a, b) => + sectorBreadth(a.sector) - sectorBreadth(b.sector) || + geographyBreadth(a.geography) - geographyBreadth(b.geography), + ); + return ranked[0].value; +} diff --git a/src/utils/loadData.test.tsx b/src/utils/loadData.test.tsx index b70ebfe5..bc6e0ab4 100644 --- a/src/utils/loadData.test.tsx +++ b/src/utils/loadData.test.tsx @@ -36,6 +36,35 @@ afterEach(() => { /* ---------------------------- * decideIncludeInvalid unit tests * ---------------------------- */ +describe("isViteDev", () => { + // This gates developer-facing logging, so the case that matters is the one + // where it must stay quiet: a production build. Vite statically replaces + // `import.meta.env.DEV` with false there, which vi.stubEnv can simulate — + // note the globalThis shim used below for decideIncludeInvalid cannot, because + // vitest supplies a real import.meta.env and readViteEnv prefers it. + it("is true when import.meta.env.DEV is set, as on the dev server", async () => { + vi.stubEnv("DEV", true); + const { isViteDev } = await importModule(); + expect(isViteDev()).toBe(true); + }); + + it("is false in a production build", async () => { + vi.stubEnv("DEV", false); + const { isViteDev } = await importModule(); + expect(isViteDev()).toBe(false); + }); + + it("ignores the flags decideIncludeInvalid reads", async () => { + // The two answer different questions; VITE_INCLUDE_INVALID must not make + // a production build report itself as dev. + vi.stubEnv("DEV", false); + vi.stubEnv("VITE_INCLUDE_INVALID", "true"); + vi.stubEnv("NODE_ENV", "development"); + const { isViteDev } = await importModule(); + expect(isViteDev()).toBe(false); + }); +}); + describe("decideIncludeInvalid", () => { it("returns false in production regardless of env flags", async () => { vi.stubEnv("NODE_ENV", "production"); diff --git a/src/utils/loadData.ts b/src/utils/loadData.ts index b57077d0..17e4ea1e 100644 --- a/src/utils/loadData.ts +++ b/src/utils/loadData.ts @@ -28,12 +28,13 @@ type ViteEnv = } | undefined; -// Decider reads both Vite and Node envs so it works in browser & Node contexts. -export function decideIncludeInvalid(): boolean { - // Try to read a Vite-like env if present (works in browser/preview and in tests with a shim) - // Access to import.meta must not use `typeof import` (keyword) — esbuild will error. +/** + * Read a Vite-like env if one is present — the browser/preview build, or a test + * shim. Access to import.meta must not use `typeof import` (keyword), which + * esbuild rejects, hence the try/catch. + */ +function readViteEnv(): ViteEnv { let viteEnv: ViteEnv = undefined; - try { // @ts-expect-error: import.meta is not defined in Node tests viteEnv = import.meta?.env; @@ -45,6 +46,29 @@ export function decideIncludeInvalid(): boolean { viteEnv = (globalThis as typeof globalThis & { import?: ImportMetaEnvShim }) .import?.meta?.env; } + return viteEnv; +} + +/** + * True only when we are confidently running the Vite dev server. + * + * Deliberately conservative: Vite replaces `import.meta.env.DEV` with `false` in + * production builds, and in Node/vitest there is no `import.meta.env` at all, so + * both of those read as "not dev". That is the behaviour wanted for gating + * developer-facing diagnostics — silent in production, and silent under test + * (where src/test/failOnReactWarnings.ts spies on console.warn). + * + * Note this cannot be derived from `decideIncludeInvalid()`: that answers a + * different question (whether to keep schema-invalid pathways) and returns false + * in dev unless VITE_INCLUDE_INVALID is set. + */ +export function isViteDev(): boolean { + return !!readViteEnv()?.DEV; +} + +// Decider reads both Vite and Node envs so it works in browser & Node contexts. +export function decideIncludeInvalid(): boolean { + const viteEnv = readViteEnv(); const viteDev = !!viteEnv?.DEV; const viteFlag = diff --git a/src/utils/searchUtils.scopedFacets.test.ts b/src/utils/searchUtils.scopedFacets.test.ts new file mode 100644 index 00000000..8c0d933f --- /dev/null +++ b/src/utils/searchUtils.scopedFacets.test.ts @@ -0,0 +1,389 @@ +import { describe, it, expect } from "vitest"; +import { filterPathways, getGlobalFacetOptions } from "./searchUtils"; +import type { FiltersWithArrays } from "./searchUtils"; +import { ABSENT_FILTER_TOKEN } from "./absent"; +import type { PathwayMetadataType } from "../types"; + +/** + * Coverage for the two keyFeature-backed search facets over v2 scoped entries + * (#858): `emissionsTrajectory` and `policyAmbition`. + * + * These arms had no test coverage at all before v2 — no `filterPathways` test + * passed either filter — which is why the v1->v2 shape change broke them + * silently: `concrete.includes(v)` against an array is simply always false, so + * selecting either facet quietly returned zero pathways. + * + * A pathway matches on the value at the scope the user is looking at; holding it + * only at some *other* scope is not a match. The sector axis uses containment; the + * geography axis uses a non-empty ISO intersection, because the query vocabulary + * and each publication's region membership are different lists by design (#783). + * Fallback ranking is #869's job, not this layer's. + */ + +type Entry = { sector: string; geography: string; value: string }; + +function pathway( + id: string, + opts: { + sectors?: string[]; + emissionsTrajectory?: Entry[]; + policyAmbition?: Entry[]; + }, +): PathwayMetadataType { + return { + id, + name: { full: id }, + sectors: (opts.sectors ?? ["Power", "Steel"]).map((name) => ({ + name, + technologies: [], + })), + geography: { + regions: { "South East Asia": ["ID", "TH", "VN"] }, + country: ["US"], + }, + metric: [], + keyFeatures: { + emissionsTrajectory: opts.emissionsTrajectory ?? [], + policyAmbition: opts.policyAmbition ?? [], + }, + } as unknown as PathwayMetadataType; +} + +const e = (sector: string, geography: string, value: string): Entry => ({ + sector, + geography, + value, +}); + +const ids = (list: PathwayMetadataType[]) => list.map((p) => p.id).sort(); + +describe("emissionsTrajectory facet over scoped entries", () => { + const wide = pathway("wide", { + emissionsTrajectory: [ + e("across sectors", "Global", "Significant decrease"), + ], + }); + const perSector = pathway("perSector", { + emissionsTrajectory: [ + e("Power", "Global", "Significant decrease"), + e("Steel", "Global", "Minor decrease"), + ], + }); + const regional = pathway("regional", { + emissionsTrajectory: [ + e("across sectors", "South East Asia", "Significant decrease"), + ], + }); + const empty = pathway("empty", { emissionsTrajectory: [] }); + const all = [wide, perSector, regional, empty]; + + it("matches a value held at the widest scope when nothing is narrowed", () => { + const filters: FiltersWithArrays = { + emissionsTrajectory: ["Significant decrease"], + }; + expect(ids(filterPathways(all, filters))).toEqual([ + "perSector", + "regional", + "wide", + ]); + }); + + it("respects the user's sector: excludes a pathway whose value at that sector differs", () => { + // perSector holds "Significant decrease" for Power but "Minor decrease" for + // Steel. Filtering to Steel must not match it, even though the value exists + // elsewhere in the pathway. This is the whole point of the scope check. + const filters: FiltersWithArrays = { + sector: ["Steel"], + emissionsTrajectory: ["Significant decrease"], + }; + const result = ids(filterPathways(all, filters)); + expect(result).not.toContain("perSector"); + expect(result).toEqual(["regional", "wide"]); + }); + + it("matches that same pathway when the user asks about the sector it holds", () => { + const filters: FiltersWithArrays = { + sector: ["Power"], + emissionsTrajectory: ["Significant decrease"], + }; + expect(ids(filterPathways(all, filters))).toContain("perSector"); + }); + + it("matches a regional entry from a country inside that region", () => { + const filters: FiltersWithArrays = { + geography: ["TH"], + emissionsTrajectory: ["Significant decrease"], + }; + expect(ids(filterPathways(all, filters))).toContain("regional"); + }); + + it("does not match a regional entry from a country outside that region", () => { + // "regional" only has South East Asia data, which does not include the US. + // (The geography facet would also exclude it, but this pins the scope check + // independently — the pathway does declare US coverage.) + const filters: FiltersWithArrays = { + geography: ["US"], + emissionsTrajectory: ["Significant decrease"], + }; + expect(ids(filterPathways(all, filters))).not.toContain("regional"); + }); + + it("treats an empty entry list as the absent bucket", () => { + expect( + ids(filterPathways(all, { emissionsTrajectory: [ABSENT_FILTER_TOKEN] })), + ).toEqual(["empty"]); + }); + + it("does not match the absent bucket when a value is present", () => { + const filters: FiltersWithArrays = { + emissionsTrajectory: [ABSENT_FILTER_TOKEN], + }; + expect(ids(filterPathways(all, filters))).not.toContain("wide"); + }); + + it("passes every pathway through when the facet is not selected", () => { + expect(ids(filterPathways(all, {}))).toEqual([ + "empty", + "perSector", + "regional", + "wide", + ]); + }); + + it("ANY mode matches a pathway holding either selected value", () => { + const filters: FiltersWithArrays = { + emissionsTrajectory: ["Minor decrease", "Moderate decrease"], + modes: { emissionsTrajectory: "ANY" }, + }; + expect(ids(filterPathways(all, filters))).toEqual(["perSector"]); + }); + + it("ALL mode requires every selected value, which multi-scope data can satisfy", () => { + // Newly meaningful in v2: one pathway can genuinely hold two values at once. + const filters: FiltersWithArrays = { + emissionsTrajectory: ["Significant decrease", "Minor decrease"], + modes: { emissionsTrajectory: "ALL" }, + }; + expect(ids(filterPathways(all, filters))).toEqual(["perSector"]); + }); + + it("ALL mode excludes a pathway holding only one of the selected values", () => { + const filters: FiltersWithArrays = { + emissionsTrajectory: ["Significant decrease", "Minor decrease"], + modes: { emissionsTrajectory: "ALL" }, + }; + expect(ids(filterPathways(all, filters))).not.toContain("wide"); + }); + + it("across sectors does not answer a sector the pathway never declares", () => { + const powerOnly = pathway("powerOnly", { + sectors: ["Power"], + emissionsTrajectory: [ + e("across sectors", "Global", "Significant decrease"), + ], + }); + const filters: FiltersWithArrays = { + sector: ["Steel"], + emissionsTrajectory: ["Significant decrease"], + }; + // The sector facet excludes it too; asserted here so the scope rule is + // pinned independently of that. + expect(ids(filterPathways([powerOnly], filters))).toEqual([]); + }); +}); + +describe("policyAmbition facet over scoped entries", () => { + const p = pathway("p", { + policyAmbition: [ + e("Power", "Global", "High ambition policies"), + e("Steel", "Global", "Current/legislated policies"), + ], + }); + + it("respects the user's sector, same as emissionsTrajectory", () => { + expect( + ids( + filterPathways([p], { + sector: ["Steel"], + policyAmbition: ["High ambition policies"], + }), + ), + ).toEqual([]); + expect( + ids( + filterPathways([p], { + sector: ["Power"], + policyAmbition: ["High ambition policies"], + }), + ), + ).toEqual(["p"]); + }); + + it("applies independently of the emissionsTrajectory facet", () => { + const filters: FiltersWithArrays = { + policyAmbition: ["Current/legislated policies"], + emissionsTrajectory: ["Significant decrease"], + }; + // emissionsTrajectory is empty on this pathway, so the conjunction fails. + expect(ids(filterPathways([p], filters))).toEqual([]); + }); +}); + +describe("region filters vs publication region membership", () => { + // Regression for the containment bug. The fixture pathway names its region + // "South East Asia" with members [ID, TH, VN]; the filter vocabulary's token is + // "Southeast Asia" with 11 codes. The lists differ in both spelling and + // membership, which is normal (#783) — and under strict containment it meant a + // region filter plus a keyFeature facet returned nothing, even though the + // geography facet alone (which has always used overlap) kept the pathway. + const regional = pathway("regional", { + emissionsTrajectory: [ + e("across sectors", "South East Asia", "Significant decrease"), + ], + }); + + it("matches when the query region overlaps the entry's region", () => { + expect( + ids( + filterPathways([regional], { + geography: ["Southeast Asia"], + emissionsTrajectory: ["Significant decrease"], + }), + ), + ).toEqual(["regional"]); + }); + + it("agrees with the geography facet applied on its own", () => { + // The two must not disagree about the same pathway — that divergence was the + // bug, and it only showed up once a second filter was added. + const geoOnly = ids( + filterPathways([regional], { geography: ["Southeast Asia"] }), + ); + const combined = ids( + filterPathways([regional], { + geography: ["Southeast Asia"], + emissionsTrajectory: ["Significant decrease"], + }), + ); + expect(geoOnly).toEqual(["regional"]); + expect(combined).toEqual(geoOnly); + }); + + it("still excludes a region that shares no countries with the entry", () => { + expect( + ids( + filterPathways([regional], { + geography: ["North America"], + emissionsTrajectory: ["Significant decrease"], + }), + ), + ).toEqual([]); + }); +}); + +describe("Sector=None combined with a scoped facet", () => { + // Regression for the case Copilot flagged: selecting the "None" sector bucket + // alongside a keyFeature facet used to return nothing, because the ABSENT token + // was compared as though it were a sector name and filtered out every entry. + const noSectors = { + id: "no-sectors", + name: { full: "No Sectors" }, + sectors: [], + geography: { global: true }, + metric: [], + keyFeatures: { + emissionsTrajectory: [ + e("across sectors", "Global", "Significant decrease"), + ], + policyAmbition: [], + }, + } as unknown as PathwayMetadataType; + + it("matches on the sector bucket alone", () => { + expect( + ids(filterPathways([noSectors], { sector: [ABSENT_FILTER_TOKEN] })), + ).toEqual(["no-sectors"]); + }); + + it("still matches when combined with the keyFeature facet", () => { + expect( + ids( + filterPathways([noSectors], { + sector: [ABSENT_FILTER_TOKEN], + emissionsTrajectory: ["Significant decrease"], + }), + ), + ).toEqual(["no-sectors"]); + }); + + it("does not match a value the pathway does not hold", () => { + expect( + ids( + filterPathways([noSectors], { + sector: [ABSENT_FILTER_TOKEN], + emissionsTrajectory: ["Minor decrease"], + }), + ), + ).toEqual([]); + }); + + it("behaves the same for the geography None bucket", () => { + expect( + ids( + filterPathways([noSectors], { + geography: [ABSENT_FILTER_TOKEN], + emissionsTrajectory: ["Significant decrease"], + }), + ), + ).toEqual([]); // geography facet itself excludes it: this pathway IS global + }); +}); + +describe("getGlobalFacetOptions over scoped entries", () => { + const all = [ + pathway("a", { + emissionsTrajectory: [ + e("across sectors", "Global", "Significant decrease"), + ], + policyAmbition: [e("across sectors", "Global", "High ambition policies")], + }), + pathway("b", { + emissionsTrajectory: [ + e("Power", "Global", "Minor decrease"), + e("Steel", "Global", "Significant decrease"), + ], + }), + ]; + + it("lists the union of values across every scope, deduplicated", () => { + const { emissionsTrajectoryOptions } = getGlobalFacetOptions(all); + const values = emissionsTrajectoryOptions + .map((o) => o.value) + .filter((v) => v !== ABSENT_FILTER_TOKEN); + expect([...values].sort()).toEqual([ + "Minor decrease", + "Significant decrease", + ]); + }); + + it("never emits an option built from an entry object", () => { + // The v1->v2 break turned these options into "[object Object]" because the + // entries were passed through verbatim instead of their values. + const { emissionsTrajectoryOptions, policyAmbitionOptions } = + getGlobalFacetOptions(all); + for (const opt of [ + ...emissionsTrajectoryOptions, + ...policyAmbitionOptions, + ]) { + expect(String(opt.label)).not.toContain("object"); + expect(String(opt.value)).not.toContain("object"); + } + }); + + it("offers the absent bucket when a pathway has no entries for the field", () => { + const { policyAmbitionOptions } = getGlobalFacetOptions(all); + expect(policyAmbitionOptions.map((o) => o.value)).toContain( + ABSENT_FILTER_TOKEN, + ); + }); +}); diff --git a/src/utils/searchUtils.ts b/src/utils/searchUtils.ts index 1011ecc2..4c7ef059 100644 --- a/src/utils/searchUtils.ts +++ b/src/utils/searchUtils.ts @@ -14,6 +14,7 @@ import { selectedGeographyToISO, } from "./filterRegions"; import { matchesOptionalFacetAny, matchesOptionalFacetAll } from "./facets"; +import { entryValues, valuesInScope } from "./keyFeatureScope"; import { ABSENT_FILTER_TOKEN } from "./absent"; import { buildOptionsFromValues, hasAbsent, withAbsentOption } from "./facets"; import { sortPathwayType } from "./sortUtils"; @@ -81,15 +82,28 @@ export function getGlobalFacetOptions(pathways: PathwayMetadataType[]) { pathways.map((d) => d.metric).flat(), ); - // emissionsTrajectory - const emissionsTrajectoryOptions = buildOptionsFromValues( - pathways.map((d) => d.keyFeatures.emissionsTrajectory).flat(), - ); + // emissionsTrajectory / policyAmbition (#858: scoped entries, not scalars). + // Options are the union of the values across every scope a pathway declares — + // deliberately unfiltered, so the dropdown lists everything selectable rather + // than shifting as the user narrows sector/geography. + // + // The ABSENT bucket has to be added explicitly, as it is for sectors below. In + // v1 a pathway missing the field contributed `undefined`, which + // buildOptionsFromValues read as absent; in v2 an empty entries array + // contributes *no* elements, so nothing would signal absence and the "None" + // option would vanish even though the filter still honours the token. + const scopedFacetOptions = ( + field: "emissionsTrajectory" | "policyAmbition", + ) => { + const values = pathways.map((d) => entryValues(d.keyFeatures?.[field])); + return withAbsentOption( + buildOptionsFromValues(values.flat()), + values.some((v) => v.length === 0), + ); + }; - // Policy ambition - const policyAmbitionOptions = buildOptionsFromValues( - pathways.map((d) => d.keyFeatures.policyAmbition).flat(), - ); + const emissionsTrajectoryOptions = scopedFacetOptions("emissionsTrajectory"); + const policyAmbitionOptions = scopedFacetOptions("policyAmbition"); const dataAvailabilityOptions = buildOptionsFromValues( pathways.map((d) => availabilityFor(d)).flat(), @@ -435,68 +449,45 @@ export const filterPathways = ( if (!ok) return false; } - // emissionsTrajectory filter + // emissionsTrajectory / policyAmbition filters (#858). + // + // These two are keyFeature fields *and* search facets. In v1 each held one + // scalar; in v2 each holds scoped {sector, geography, value} entries, so a + // pathway can carry several values at once and the question becomes "which + // value, at which scope?". + // + // Answer, per the product decision: the value at the scope the user is + // looking at. `valuesInScope` narrows the entries to those whose scope + // contains the active sector/geography selection, and matching then works + // exactly like the sector and metric facets above — an ANY/ALL match over a + // list of values, with an empty list treated as the ABSENT bucket. A pathway + // holding the selected value only at some *other* scope is not a match. + // + // Containment only: no fallback ranking, no "how far did we broaden" cost. + // That is #869, and it is what will turn a non-match at the queried scope + // into a ranked broader-scope match rather than an exclusion. { - const selected = toArray(filters.emissionsTrajectory); - if (selected.length) { - const hasAbsent = selected.includes(ABSENT_FILTER_TOKEN); - const concrete = selected.filter((t) => t !== ABSENT_FILTER_TOKEN); - const v = pathway.keyFeatures?.emissionsTrajectory ?? null; - const mode = pickMode("emissionsTrajectory", filters.modes); - let ok = true; - - if (mode === "ANY") { - ok = - (v == null && hasAbsent) || - (v != null && (concrete.length ? concrete.includes(v) : false)); - } else { - // ALL: for single-valued fields, all selected tokens must hold. - // That is only possible when exactly one token is selected: - // - [ABSENT] -> v == null - // - [X] -> v == X - // Any combination (ABSENT + X, or X + Y) cannot be satisfied. - if (hasAbsent && concrete.length === 0) { - ok = v == null; - } else if (!hasAbsent && concrete.length === 1) { - ok = v != null && v === concrete[0]; - } else { - ok = false; - } - } - if (!ok) return false; - } - } - - // policyAmbition filter - { - const selected = toArray(filters.policyAmbition); - if (selected.length) { - const hasAbsent = selected.includes(ABSENT_FILTER_TOKEN); - const concrete = selected.filter((t) => t !== ABSENT_FILTER_TOKEN); - const v = pathway.keyFeatures?.policyAmbition ?? null; - const mode = pickMode("policyAmbition", filters.modes); - let ok = true; + const scopeQuery = { + sectors: toArray(filters.sector), + geographies: toArray(filters.geography), + }; + const scopedFacetOk = ( + facet: "emissionsTrajectory" | "policyAmbition", + ): boolean => { + const selected = toArray(filters[facet]); + if (selected.length === 0) return true; + const values = valuesInScope( + pathway.keyFeatures?.[facet], + scopeQuery, + pathway, + ); + return pickMode(facet, filters.modes) === "ALL" + ? matchesOptionalFacetAll(selected, values, (s) => s) + : matchesOptionalFacetAny(selected, values, (s) => s); + }; - if (mode === "ANY") { - ok = - (v == null && hasAbsent) || - (v != null && (concrete.length ? concrete.includes(v) : false)); - } else { - // ALL: for single-valued fields, all selected tokens must hold. - // That is only possible when exactly one token is selected: - // - [ABSENT] -> v == null - // - [X] -> v == X - // Any combination (ABSENT + X, or X + Y) cannot be satisfied. - if (hasAbsent && concrete.length === 0) { - ok = v == null; - } else if (!hasAbsent && concrete.length === 1) { - ok = v != null && v === concrete[0]; - } else { - ok = false; - } - } - if (!ok) return false; - } + if (!scopedFacetOk("emissionsTrajectory")) return false; + if (!scopedFacetOk("policyAmbition")) return false; } // dataAvailability filter diff --git a/src/utils/timeseriesAvailability.test.ts b/src/utils/timeseriesAvailability.test.ts index 8cbb97e7..a649f68b 100644 --- a/src/utils/timeseriesAvailability.test.ts +++ b/src/utils/timeseriesAvailability.test.ts @@ -158,5 +158,17 @@ describe("pathwayToolAvailability", () => { expect(av.hasMetric("Capacity")).toBe(false); expect(av.hasGeography("Global")).toBe(false); }); + + it("tolerates a sector the taxonomy defines without a metrics axis", () => { + // Steel and Aviation are in SECTORS_BY_KEY for their data-availability + // vocabularies but define no timeseries `metrics`. Indexing that + // undefined axis used to throw, which would have broken the detail page + // the day a Steel series landed. + const av = pathwayToolAvailability([ + { summary: { sectors: ["steel"], metrics: ["capacity"] } }, + ]); + expect(av.hasSector("Steel")).toBe(true); + expect(av.hasMetric("Capacity")).toBe(false); + }); }); }); diff --git a/src/utils/timeseriesAvailability.ts b/src/utils/timeseriesAvailability.ts index c438ce21..f4b812d2 100644 --- a/src/utils/timeseriesAvailability.ts +++ b/src/utils/timeseriesAvailability.ts @@ -38,7 +38,9 @@ function metricDisplayNames(summary: unknown): Set { const sectorDef = SECTORS_BY_KEY[sectorKey]; if (!sectorDef) continue; for (const metricKey of s.metrics ?? []) { - const metricDef = sectorDef.metrics[metricKey]; + // `metrics` is optional: Steel and Aviation are defined for their + // data-availability vocabularies but plot no timeseries yet. + const metricDef = sectorDef.metrics?.[metricKey]; if (metricDef) names.add(metricDef.displayName); } } diff --git a/src/utils/timeseriesTaxonomy.test.ts b/src/utils/timeseriesTaxonomy.test.ts new file mode 100644 index 00000000..9755b062 --- /dev/null +++ b/src/utils/timeseriesTaxonomy.test.ts @@ -0,0 +1,322 @@ +import { describe, it, expect } from "vitest"; +import { + availabilityMetricBelongsToSector, + availabilityMetricsForSector, + metricBelongsToSector, + metricsForSector, + POWER_SECTOR_DEFINITION, + SECTORS_BY_KEY, + segmentBelongsToSector, + segmentsForSector, + technologiesForSector, + technologyBelongsToSector, + UNSEGMENTED, +} from "./timeseriesTaxonomy"; +import sectorSchema from "../schema/common/sector.v1.json" with { type: "json" }; +import technologySchema from "../schema/common/technology.v1.json" with { type: "json" }; +import metricSchema from "../schema/common/metric.v1.json" with { type: "json" }; +import segmentSchema from "../schema/common/sectorSegment.v1.json" with { type: "json" }; +import availabilityMetricSchema from "../schema/common/dataAvailabilityMetric.v1.json" with { type: "json" }; + +const SECTOR_NAMES: string[] = sectorSchema.$defs.displayName.enum; +const TECHNOLOGY_NAMES: string[] = technologySchema.$defs.displayName.enum; +const METRIC_NAMES: string[] = metricSchema.$defs.displayName.enum; +const SEGMENT_NAMES: string[] = segmentSchema.$defs.displayName.enum; +const AVAILABILITY_METRIC_NAMES: string[] = + availabilityMetricSchema.$defs.displayName.enum; + +describe("technologiesForSector", () => { + it("accepts the two technologies cookbook 0020 added to Power", () => { + // Both were added to technology.v1.json's enum by the #858 pass but not + // here, so AJV accepted them while technologyBelongsToSector said "no" -- + // which silently dropped them from JETP-CIPP-2023 on import. + expect(technologyBelongsToSector("Geothermal", "Power")).toBe("yes"); + expect(technologyBelongsToSector("Energy storage", "Power")).toBe("yes"); + }); + + it("resolves Power to its twelve technologies", () => { + expect(technologiesForSector("Power")).toEqual([ + "Biomass", + "Coal", + "Energy storage", + "Gas", + "Geothermal", + "Hydro", + "Nuclear", + "Oil", + "Other", + "Renewables", + "Solar", + "Wind", + ]); + }); + + it("derives the list from POWER_SECTOR_DEFINITION rather than duplicating it", () => { + // Drift guard: the metadata-side allowlist and the chart-side taxonomy are + // the same data, so adding a technology to the definition must be the only + // edit needed. Same intent as the wrapper-consistency tests in + // pathwayMetadata.v2.test.ts. + expect(technologiesForSector("Power")).toEqual( + Object.values(POWER_SECTOR_DEFINITION.technologies ?? {}).map( + (t) => t.displayName, + ), + ); + }); + + it("returns undefined -- not [] -- for a sector with no definition", () => { + // The distinction is load-bearing: validateScopedEntries rejects a non-empty + // list here, and could not tell "defined and empty" from "undefined" if this + // collapsed to []. + expect(technologiesForSector("Steel")).toBeUndefined(); + }); + + it("returns undefined for a string that is not a sector at all", () => { + expect(technologiesForSector("Narnia")).toBeUndefined(); + }); + + it("keys on display name, not on the camelCase taxonomy key", () => { + // SECTORS_BY_KEY is keyed "power"; metadata says "Power". Passing the key + // must not accidentally work, or data could name sectors either way. + expect(technologiesForSector("power")).toBeUndefined(); + }); + + it("only defines technologies for sectors the metadata schema knows", () => { + for (const sector of Object.values(SECTORS_BY_KEY)) { + expect(SECTOR_NAMES).toContain(sector.displayName); + } + }); + + it("only names technologies the metadata schema allows", () => { + // A displayName here that the enum lacks would be permanently unusable: AJV + // would reject the data before this check ever saw it. + for (const sector of Object.values(SECTORS_BY_KEY)) { + for (const tech of Object.values(sector.technologies ?? {})) { + expect(TECHNOLOGY_NAMES).toContain(tech.displayName); + } + } + }); +}); + +describe("technologyBelongsToSector", () => { + it("says yes for a technology of a defined sector", () => { + expect(technologyBelongsToSector("Solar", "Power")).toBe("yes"); + }); + + it("says no for a technology outside a defined sector's list", () => { + expect(technologyBelongsToSector("Hydrogen Use", "Power")).toBe("no"); + }); + + it("says unknown -- not no -- for a sector with no definition", () => { + // The difference matters to non-validation callers: #869's technology axis + // would silently drop every Steel pathway if this answered "no". + expect(technologyBelongsToSector("Hydrogen Use", "Steel")).toBe("unknown"); + expect(technologyBelongsToSector("Solar", "Steel")).toBe("unknown"); + }); + + it("says unknown for an unrecognised sector", () => { + expect(technologyBelongsToSector("Solar", "Narnia")).toBe("unknown"); + }); + + it("is exact about names, not fuzzy", () => { + expect(technologyBelongsToSector("solar", "Power")).toBe("no"); + expect(technologyBelongsToSector("Solar Power", "Power")).toBe("no"); + }); + + it("classifies every enum technology under Power as yes or no, never unknown", () => { + for (const tech of TECHNOLOGY_NAMES) { + expect(technologyBelongsToSector(tech, "Power")).not.toBe("unknown"); + } + }); +}); + +describe("metricsForSector / metricBelongsToSector (#870)", () => { + it("resolves Power to its five metrics", () => { + expect(metricsForSector("Power")).toEqual([ + "Absolute Emissions", + "Capacity", + "Emissions Intensity", + "Generation", + "Technology Mix", + ]); + }); + + it("derives the list from POWER_SECTOR_DEFINITION rather than duplicating it", () => { + expect(metricsForSector("Power")).toEqual( + Object.values(POWER_SECTOR_DEFINITION.metrics ?? {}).map( + (m) => m.displayName, + ), + ); + }); + + it("covers the whole metric enum for Power", () => { + // Worth pinning: it is why the sector/metric check cannot currently reject + // a real Power row, and why it only becomes live for production data once a + // second sector defines its metrics. + expect([...(metricsForSector("Power") ?? [])].sort()).toEqual( + [...METRIC_NAMES].sort(), + ); + }); + + it("returns undefined for a sector with no metrics defined", () => { + expect(metricsForSector("Steel")).toBeUndefined(); + }); + + it("says yes for a metric of a defined sector", () => { + expect(metricBelongsToSector("Capacity", "Power")).toBe("yes"); + }); + + it("says no for a metric outside a defined sector's list", () => { + expect(metricBelongsToSector("Water Use", "Power")).toBe("no"); + }); + + it("says unknown for a sector with no metrics defined", () => { + // validateScopedEntries lets "unknown" pass for this axis, unlike the other + // two — see the comment on that check. + expect(metricBelongsToSector("Capacity", "Steel")).toBe("unknown"); + }); + + it("only names metrics the schema allows", () => { + for (const sector of Object.values(SECTORS_BY_KEY)) { + for (const metric of Object.values(sector.metrics ?? {})) { + expect(METRIC_NAMES).toContain(metric.displayName); + } + } + }); +}); + +describe("availabilityMetricsForSector / availabilityMetricBelongsToSector (#870)", () => { + it("is a superset of the metadata metric list for Power", () => { + // The whole point of the second axis (register item D16): the availability + // row key adds the four metrics a pathway reports but we do not plot. + const avail = availabilityMetricsForSector("Power") ?? []; + for (const extra of [ + "Transmission lines", + "Technology cost", + "Investment requirement", + "Asset lifetime", + ]) { + expect(avail).toContain(extra); + } + expect(avail).toHaveLength(9); + }); + + it("does not carry Storage capacity, which D16 dropped", () => { + expect(availabilityMetricsForSector("Power")).not.toContain( + "Storage capacity", + ); + }); + + it("answers for Steel and Aviation, which the metadata axis does not", () => { + expect(availabilityMetricsForSector("Steel")).toHaveLength(10); + expect(availabilityMetricsForSector("Aviation")).toHaveLength(13); + // The metadata axis is still Power-only, which is what keeps the search + // facet unchanged. + expect(metricsForSector("Steel")).toBeUndefined(); + expect(metricsForSector("Aviation")).toBeUndefined(); + }); + + it("scopes a metric to its own sector", () => { + expect(availabilityMetricBelongsToSector("Scrap share", "Steel")).toBe( + "yes", + ); + expect(availabilityMetricBelongsToSector("Scrap share", "Power")).toBe( + "no", + ); + expect(availabilityMetricBelongsToSector("Capacity", "Power")).toBe("yes"); + }); + + it("says unknown for a sector the cookbook has not reached", () => { + // Keeps dataAvailability authorable for the other twelve sectors. + expect(availabilityMetricBelongsToSector("Capacity", "Cement")).toBe( + "unknown", + ); + }); + + it("only names metrics the schema allows", () => { + // A displayName the enum lacks would be permanently unusable: AJV rejects + // the row before this check sees it. + for (const sector of Object.values(SECTORS_BY_KEY)) { + for (const metric of Object.values(sector.availabilityMetrics ?? {})) { + expect(AVAILABILITY_METRIC_NAMES).toContain(metric.displayName); + } + } + }); +}); + +describe("segmentsForSector / segmentBelongsToSector (#870)", () => { + it("resolves Power to its four segments", () => { + expect(segmentsForSector("Power")).toEqual([ + "Fuel extraction and processing", + "Power generation", + "Energy storage", + "Transmission and distribution", + ]); + }); + + it("resolves Steel and Aviation too, not just Power", () => { + // All three sectors the cookbook defines segments for (#858). + expect(segmentsForSector("Steel")).toEqual([ + "Downstream", + "Fuel extraction and processing", + "Ironmaking", + "Mining", + "Steelmaking", + ]); + expect(segmentsForSector("Aviation")).toEqual([ + "Freight transport", + "Passenger transport", + "Upstream energy and fuels", + ]); + }); + + it("returns undefined for a sector with no segments defined", () => { + // Cement is one of the twelve the cookbook has not reached yet, and the + // undefined is load-bearing -- see vocabularyFor. + expect(segmentsForSector("Cement")).toBeUndefined(); + }); + + it("excludes the universal sentinel from the defined list", () => { + // UNSEGMENTED is legal everywhere, so listing it under Power would imply it + // is Power's in particular. + expect(segmentsForSector("Power")).not.toContain(UNSEGMENTED); + }); + + it("says yes for a segment of a defined sector", () => { + expect(segmentBelongsToSector("Energy storage", "Power")).toBe("yes"); + }); + + it("says no for a segment outside a defined sector's list", () => { + expect(segmentBelongsToSector("Refining", "Power")).toBe("no"); + }); + + it("says unknown for a named segment under an undefined sector", () => { + expect(segmentBelongsToSector("Energy storage", "Cement")).toBe("unknown"); + }); + + it("says no for a segment of the wrong defined sector", () => { + // Now that three sectors have segments, the lists can actually disagree: + // Ironmaking is Steel's, not Power's. + expect(segmentBelongsToSector("Ironmaking", "Power")).toBe("no"); + expect(segmentBelongsToSector("Energy storage", "Steel")).toBe("no"); + }); + + it("says yes to the sentinel under every sector, defined or not", () => { + // This is what keeps dataAvailability authorable before a sector's segments + // are written down. + for (const name of [...SECTOR_NAMES, "Narnia"]) { + expect(segmentBelongsToSector(UNSEGMENTED, name)).toBe("yes"); + } + }); + + it("only names segments the schema allows", () => { + for (const sector of Object.values(SECTORS_BY_KEY)) { + for (const segment of Object.values(sector.segments ?? {})) { + expect(SEGMENT_NAMES).toContain(segment.displayName); + } + } + }); + + it("keeps the sentinel in the schema enum", () => { + expect(SEGMENT_NAMES).toContain(UNSEGMENTED); + }); +}); diff --git a/src/utils/timeseriesTaxonomy.ts b/src/utils/timeseriesTaxonomy.ts index a3adcd38..51e58dc2 100644 --- a/src/utils/timeseriesTaxonomy.ts +++ b/src/utils/timeseriesTaxonomy.ts @@ -11,11 +11,46 @@ export interface MetricDefinition { sectorScope?: string; } +/** + * A subdivision of a sector (#870): Power splits into generation, storage and + * transmission & distribution, and a metric's data availability can differ + * between them. + */ +export interface SegmentDefinition { + displayName: string; + definition: string; +} + export interface SectorDefinition { key: string; displayName: string; - technologies: Record; - metrics: Record; + /** + * Every axis below `displayName` is optional, and absent is not the same as + * empty: `vocabularyFor` returns `undefined` for an axis a sector does not + * define, which is what separates "nobody has written this sector's values + * down" from "this sector has none". Supplying `{}` to satisfy a required + * field would silently close an undefined axis — see the comment on + * `vocabularyFor`. + * + * Steel and Aviation define metrics and segments but no technologies, which + * is why `technologies` became optional alongside `segments`. + */ + technologies?: Record; + metrics?: Record; + segments?: Record; + /** + * The cookbook's "Metrics – Extended" — the row key of the Data Availability + * table, distinct from `metrics` above. + * + * Two separate axes because the cookbook carries two metric variables, which + * register item D16 records as legitimately different rather than in + * conflict. `metrics` is the pathway-level multi-select that also names the + * plotted timeseries; this adds the metrics a pathway reports but we do not + * plot (transmission lines, technology cost, investment requirement, asset + * lifetime). Folding them into `metrics` would make the timeseries taxonomy + * claim series it has no data for. + */ + availabilityMetrics?: Record; } export const POWER_SECTOR_DEFINITION: SectorDefinition = { @@ -32,11 +67,21 @@ export const POWER_SECTOR_DEFINITION: SectorDefinition = { definition: "Electricity generation using coal combustion to produce steam that drives turbines for power.", }, + energyStorage: { + displayName: "Energy storage", + definition: + "Storing electricity for later dispatch, however the pathway breaks it down -- batteries, pumped hydro or both. The cookbook removed the separate BESS and pumped-hydro technologies in decision 0020, so a Capacity row's scope limitations are the only place that detail survives.", + }, gas: { displayName: "Gas", definition: "Electricity generation using natural gas combustion in turbines or combined-cycle plants.", }, + geothermal: { + displayName: "Geothermal", + definition: + "Electricity generation using heat drawn from the earth to raise steam.", + }, hydro: { displayName: "Hydro", definition: @@ -55,7 +100,7 @@ export const POWER_SECTOR_DEFINITION: SectorDefinition = { other: { displayName: "Other", definition: - "Electricity generation using alternative or emerging sources such as geothermal, tidal, or hydrogen.", + "Electricity generation using alternative or emerging sources such as tidal or hydrogen. Geothermal is no longer among them -- it is its own technology as of cookbook decision 0020.", }, renewables: { displayName: "Renewables", @@ -73,6 +118,31 @@ export const POWER_SECTOR_DEFINITION: SectorDefinition = { "Electricity generation via wind turbines, both onshore and offshore.", }, }, + // Keys alphabetical, matching the technologies and metrics around them. The + // order surfaces through segmentsForSector, which feeds validateScopes' + // "legal values" error messages. + segments: { + fuelExtractionAndProcessing: { + displayName: "Fuel extraction and processing", + definition: + "Extraction and processing of the fuels burned to generate electricity.", + }, + powerGeneration: { + displayName: "Power generation", + definition: + "Generation of electricity at the plant, before it reaches the grid.", + }, + storage: { + displayName: "Energy storage", + definition: + "Storing electricity for later dispatch (batteries, pumped hydro).", + }, + transmissionAndDistribution: { + displayName: "Transmission and distribution", + definition: + "Moving electricity from generators to consumers, including grid losses.", + }, + }, metrics: { absoluteEmissions: { displayName: "Absolute Emissions", @@ -105,11 +175,223 @@ export const POWER_SECTOR_DEFINITION: SectorDefinition = { sectorScope: "Power generation", }, }, + /* + Power's "Metrics – Extended" (cookbook data_availability/metrics_extended.md): + the five in `metrics` above plus the four a pathway reports but we do not + plot. Spellings are the cookbook's, which is why `Emissions intensity` here + differs in case from `Emissions Intensity` in `metrics` -- #858 accepts the + inconsistency for now. + */ + availabilityMetrics: { + absoluteEmissions: { + displayName: "Absolute Emissions", + definition: + "Total greenhouse gas emissions produced, regardless of output.", + }, + assetLifetime: { + displayName: "Asset lifetime", + definition: + "Assumptions about how long high-carbon assets stay in service, and whether they retire early.", + }, + capacity: { + displayName: "Capacity", + definition: "Maximum output under ideal conditions, measured in GW.", + }, + emissionsIntensity: { + displayName: "Emissions intensity", + definition: "Greenhouse gases emitted per unit of physical output.", + }, + generation: { + displayName: "Generation", + definition: "Electricity produced over a period, typically in TWh.", + }, + investmentRequirement: { + displayName: "Investment requirement", + definition: "Capital the pathway implies, and how it is broken down.", + }, + technologyCost: { + displayName: "Technology cost", + definition: + "Cost assumptions per technology, and how they are broken down.", + }, + technologyMix: { + displayName: "Technology mix", + definition: "The breakdown of sources used for electricity generation.", + }, + transmissionLines: { + displayName: "Transmission lines", + definition: + "Transmission and distribution infrastructure, and which connections are in scope.", + }, + }, +}; + +/* + Steel and Aviation, added for #870's data-availability rows (#858). + + Neither defines `technologies` or `metrics`: the cookbook gives both sectors a + technology-coverage list and a metadata metric list, but nothing in this repo + plots their timeseries yet, and leaving those axes undefined is what keeps + `technologyBelongsToSector` answering "unknown" rather than "no" for them. Add + them when the data arrives, not before -- an empty object would close the axis. +*/ +export const STEEL_SECTOR_DEFINITION: SectorDefinition = { + key: "steel", + displayName: "Steel", + segments: { + downstream: { + displayName: "Downstream", + definition: "Casting, rolling and finishing after steel is made.", + }, + fuelExtractionAndProcessing: { + displayName: "Fuel extraction and processing", + definition: "Extraction and processing of fuels and reductants.", + }, + ironmaking: { + displayName: "Ironmaking", + definition: "Reduction of iron ore to iron, before steelmaking.", + }, + mining: { + displayName: "Mining", + definition: "Extraction of iron ore and other raw inputs.", + }, + steelmaking: { + displayName: "Steelmaking", + definition: "Conversion of iron and scrap into crude steel.", + }, + }, + availabilityMetrics: { + absoluteEmissions: { + displayName: "Absolute emissions", + definition: + "Total greenhouse gas emissions produced, regardless of output.", + }, + assetLifetime: { + displayName: "Asset lifetime", + definition: + "Assumptions about how long high-carbon assets stay in service, and whether they retire early.", + }, + emissionsIntensityPrimary: { + displayName: "Emissions intensity (primary)", + definition: + "Emissions per tonne of steel from the primary (ore-based) route.", + }, + emissionsIntensitySecondary: { + displayName: "Emissions intensity (secondary)", + definition: + "Emissions per tonne of steel from the secondary (scrap-based) route.", + }, + emissionsIntensityTotal: { + displayName: "Emissions intensity (total)", + definition: "Emissions per tonne of steel across both routes.", + }, + investmentRequirement: { + displayName: "Investment requirement", + definition: "Capital the pathway implies, and how it is broken down.", + }, + scrapShare: { + displayName: "Scrap share", + definition: + "Share of scrap in the input mix, however the pathway defines the base.", + }, + steelProductionByTechnology: { + displayName: "Steel production by technology (production route)", + definition: "Absolute production split by production route.", + }, + technologyCost: { + displayName: "Technology cost", + definition: + "Cost assumptions per technology, and how they are broken down.", + }, + technologyMix: { + displayName: "Technology mix (production by route)", + definition: "Share of production by route.", + }, + }, +}; + +export const AVIATION_SECTOR_DEFINITION: SectorDefinition = { + key: "aviation", + displayName: "Aviation", + segments: { + freightTransport: { + displayName: "Freight transport", + definition: + "Movement of cargo, including belly freight where distinguished.", + }, + passengerTransport: { + displayName: "Passenger transport", + definition: "Movement of passengers.", + }, + upstreamEnergyAndFuels: { + displayName: "Upstream energy and fuels", + definition: + "Production and supply of aviation fuels and energy carriers.", + }, + }, + availabilityMetrics: { + absoluteEmissionsWtwFreight: { + displayName: "Absolute emissions Well-to-Wheel (freight)", + definition: "Well-to-wheel emissions attributed to freight.", + }, + absoluteEmissionsWtwPassenger: { + displayName: "Absolute emissions Well-to-Wheel (passenger)", + definition: "Well-to-wheel emissions attributed to passengers.", + }, + assetLifetime: { + displayName: "Asset lifetime", + definition: + "Assumptions about how long high-carbon assets stay in service, and whether they retire early.", + }, + demandByPropulsionFreight: { + displayName: "Demand by propulsion technology (freight)", + definition: "Absolute freight demand split by propulsion technology.", + }, + demandByPropulsionPassenger: { + displayName: "Demand by propulsion technology (passenger)", + definition: "Absolute passenger demand split by propulsion technology.", + }, + demandShareByPropulsionFreight: { + displayName: "Demand share by propulsion technology (freight)", + definition: "Share of freight demand by propulsion technology.", + }, + demandShareByPropulsionPassenger: { + displayName: "Demand share by propulsion technology (passenger)", + definition: "Share of passenger demand by propulsion technology.", + }, + emissionsIntensityFreight: { + displayName: "Emissions intensity (freight)", + definition: "Emissions per unit of freight activity.", + }, + emissionsIntensityPassenger: { + displayName: "Emissions intensity (passenger)", + definition: "Emissions per unit of passenger activity.", + }, + investmentRequirement: { + displayName: "Investment requirement", + definition: "Capital the pathway implies, and how it is broken down.", + }, + technologyCost: { + displayName: "Technology cost", + definition: + "Cost assumptions per technology, and how they are broken down.", + }, + totalDemandFreight: { + displayName: "Total demand (freight)", + definition: "Total freight activity.", + }, + totalDemandPassenger: { + displayName: "Total demand (passenger)", + definition: "Total passenger activity.", + }, + }, }; // Extend this as you add more sectors: export const SECTORS_BY_KEY: Record = { power: POWER_SECTOR_DEFINITION, + steel: STEEL_SECTOR_DEFINITION, + aviation: AVIATION_SECTOR_DEFINITION, }; export class UnknownTaxonomyError extends Error { @@ -134,7 +416,7 @@ export function getMetricDefinition( metricKey: string, ): MetricDefinition { const sector = getSectorDefinition(sectorKey); - const metric = sector.metrics[metricKey]; + const metric = sector.metrics?.[metricKey]; if (!metric) { throw new UnknownTaxonomyError( `Unknown metric key "${metricKey}" for sector "${sectorKey}" in timeseries metadata definitions`, @@ -148,7 +430,7 @@ export function getTechnologyDefinition( technologyKey: string, ): TechnologyDefinition { const sector = getSectorDefinition(sectorKey); - const tech = sector.technologies[technologyKey]; + const tech = sector.technologies?.[technologyKey]; if (!tech) { throw new UnknownTaxonomyError( `Unknown technology key "${technologyKey}" for sector "${sectorKey}" in timeseries metadata definitions`, @@ -156,3 +438,170 @@ export function getTechnologyDefinition( } return tech; } + +/* -------------------------------------------------------------------------- */ +/* Sector-conditional membership (#461 technologies, #870 metrics + segments) */ +/* -------------------------------------------------------------------------- */ + +/** + * Whether some value belongs to a sector. + * + * Tri-state rather than boolean on purpose. Only one of the fifteen sectors in + * `sector.v1.json` has its vocabularies defined here, so a boolean would have to + * answer `false` for the other fourteen — indistinguishable from "definitely not + * this sector's value" and wrong for every caller that is not validation. + * `"unknown"` lets each caller choose: `validateScopedEntries` treats it as a + * failure (see below), while a search filter can treat it as a match rather than + * silently dropping every non-Power pathway. + */ +export type TaxonomyMembership = "yes" | "no" | "unknown"; + +/** @deprecated Use {@link TaxonomyMembership}; kept so #461's callers still compile. */ +export type TechnologyMembership = TaxonomyMembership; + +/** + * Sector definitions indexed by display name. + * + * `SECTORS_BY_KEY` is keyed by the camelCase key the timeseries data uses + * (`power`), while pathway metadata names sectors by their display name + * (`"Power"`, `"Oil (Upstream)"`). `displayName` is the only bridge between the + * two: there is no camelCase↔Title-Case converter in the repo, and there cannot + * be a lossless one — `capitalizeWords` is a one-way chart-label prettifier and + * could never produce `"Oil (Upstream)"` from `oilUpstream`. + */ +const SECTORS_BY_DISPLAY_NAME: ReadonlyMap = new Map( + Object.values(SECTORS_BY_KEY).map((sector) => [sector.displayName, sector]), +); + +/** The `SectorDefinition` axes that carry a sector-conditional vocabulary. */ +type VocabularyAxis = + "technologies" | "metrics" | "segments" | "availabilityMetrics"; + +/** + * The display names a sector defines on one axis, or `undefined` when that + * sector defines nothing on it. + * + * The `undefined` is load-bearing: it separates "this vocabulary is defined and + * happens to be empty" from "nobody has said what this sector's values are". + * Callers must not collapse it to `[]` — that is the difference between a + * checked sector and an unchecked one. + * + * Derived from `SECTORS_BY_KEY` rather than listed separately, so adding a + * sector definition is the single edit that opens it up to metadata as well. + */ +function vocabularyFor( + sectorDisplayName: string, + axis: VocabularyAxis, +): readonly string[] | undefined { + const defined = SECTORS_BY_DISPLAY_NAME.get(sectorDisplayName)?.[axis]; + if (!defined) return undefined; + return Object.values(defined).map((entry) => entry.displayName); +} + +/** + * Closed-by-default membership on one axis. + * + * Note this cannot distinguish "sector with no definition" from "not a sector at + * all" — both are `"unknown"`. That is fine for the callers there are: AJV has + * already checked the sector against `sector.v1.json`'s enum before validation + * reaches here, and a caller that wants to know whether a string is a sector + * should ask the schema, not the taxonomy. + */ +function membership( + value: string, + sectorDisplayName: string, + axis: VocabularyAxis, +): TaxonomyMembership { + const allowed = vocabularyFor(sectorDisplayName, axis); + if (!allowed) return "unknown"; + return allowed.includes(value) ? "yes" : "no"; +} + +/** The technologies defined for a sector (#461). */ +export function technologiesForSector( + sectorDisplayName: string, +): readonly string[] | undefined { + return vocabularyFor(sectorDisplayName, "technologies"); +} + +/** + * #461: scope technologies to their sector. Both arguments are display names, as + * they appear in pathway metadata's `sectors[]`. + */ +export function technologyBelongsToSector( + technologyDisplayName: string, + sectorDisplayName: string, +): TaxonomyMembership { + return membership(technologyDisplayName, sectorDisplayName, "technologies"); +} + +/** + * The metrics a sector's Data Availability rows may name (#870). + * + * Distinct from {@link metricsForSector}: that is the pathway-level `metric` + * vocabulary, this is the cookbook's "Metrics – Extended" row key. Register + * item D16 records the two as legitimately different lists rather than a + * conflict to reconcile. + */ +export function availabilityMetricsForSector( + sectorDisplayName: string, +): readonly string[] | undefined { + return vocabularyFor(sectorDisplayName, "availabilityMetrics"); +} + +/** #870: a `dataAvailability` row may only name a metric of its own sector. */ +export function availabilityMetricBelongsToSector( + metricDisplayName: string, + sectorDisplayName: string, +): TaxonomyMembership { + return membership( + metricDisplayName, + sectorDisplayName, + "availabilityMetrics", + ); +} + +/** The metrics defined for a sector (#870). */ +export function metricsForSector( + sectorDisplayName: string, +): readonly string[] | undefined { + return vocabularyFor(sectorDisplayName, "metrics"); +} + +/** + * #870: a `dataAvailability` row may only describe a metric its sector actually + * has. Power's five metrics are currently the whole of `metric.v1.json`'s enum, + * so this bites on the other fourteen sectors rather than on Power. + */ +export function metricBelongsToSector( + metricDisplayName: string, + sectorDisplayName: string, +): TaxonomyMembership { + return membership(metricDisplayName, sectorDisplayName, "metrics"); +} + +/** + * The segment every sector accepts, whatever its definition says (#870). + * + * Without this, closed-by-default would make `dataAvailability` unauthorable for + * the fourteen sectors whose segments nobody has written down yet — which would + * turn a validation aid into a content blocker. Mirrors the `"No information"` + * sentinel the v2 keyFeature enums already use. + */ +export const UNSEGMENTED = "No information"; + +/** The segments defined for a sector (#870), excluding the universal sentinel. */ +export function segmentsForSector( + sectorDisplayName: string, +): readonly string[] | undefined { + return vocabularyFor(sectorDisplayName, "segments"); +} + +/** #870: scope segments to their sector, with {@link UNSEGMENTED} always legal. */ +export function segmentBelongsToSector( + segmentDisplayName: string, + sectorDisplayName: string, +): TaxonomyMembership { + if (segmentDisplayName === UNSEGMENTED) return "yes"; + return membership(segmentDisplayName, sectorDisplayName, "segments"); +} diff --git a/src/utils/validateData.test.tsx b/src/utils/validateData.test.tsx index 0e53e3df..f82714eb 100644 --- a/src/utils/validateData.test.tsx +++ b/src/utils/validateData.test.tsx @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest"; import { validateDataCollect, FileEntry } from "./validateData"; import { PathwayMetadataType } from "../types"; import pathwayMetadata from "../schema/pathwayMetadata.v1.json" with { type: "json" }; +import pathwayMetadataV2 from "../schema/pathwayMetadata.v2.json" with { type: "json" }; import { commonSchemas } from "../schema/common"; function ok(entry: FileEntry | FileEntry[]) { @@ -30,7 +31,24 @@ function fail(entry: FileEntry | FileEntry[], rx?: RegExp | string) { } } +/** Same as {@link fail}, but routes the document against v2. */ +function fail2(entry: FileEntry | FileEntry[], rx?: RegExp | string) { + const arr = Array.isArray(entry) ? entry : [entry]; + const { invalid } = validateDataCollect( + arr, + pathwayMetadataV2, + commonSchemas, + ); + expect(invalid.length).toBeGreaterThan(0); + if (rx) { + const messages = invalid.flatMap((p) => p.errors).join("\n"); + expect(messages).toMatch(rx); + } +} + import basePathway from "../../testdata/valid/pathwayMetadata_standard.json" assert { type: "json" }; +import v2Full from "../../testdata/valid/pathwayMetadata_v2_full.json" assert { type: "json" }; +import v2Minimal from "../../testdata/valid/pathwayMetadata_v2_minimal.json" assert { type: "json" }; describe("pathway schema enforces expected limits", () => { it("accepts a valid object", () => { @@ -287,3 +305,241 @@ describe("pathway schema enforces expected limits", () => { ); }); }); + +describe("v1 and v2 documents coexist (#858)", () => { + const v1Entry: FileEntry = { name: "v1.json", data: basePathway }; + const v2Entry: FileEntry = { name: "v2.json", data: v2Full }; + const v2MinEntry: FileEntry = { name: "v2-min.json", data: v2Minimal }; + + it("validates a v1 document against v1", () => { + const { valid, invalid } = validateDataCollect( + [v1Entry], + pathwayMetadata, + commonSchemas, + ); + expect(invalid).toHaveLength(0); + expect(valid).toHaveLength(1); + }); + + it("validates v2 documents against v2, including all-empty keyFeatures", () => { + const { valid, invalid } = validateDataCollect( + [v2Entry, v2MinEntry], + pathwayMetadataV2, + commonSchemas, + ); + expect(invalid).toHaveLength(0); + expect(valid).toHaveLength(2); + }); + + it("accepts a declared three-letter region label as a scope token", () => { + // The cookbook says to keep a multi-country region's label exactly as the + // publication writes it, and model regions are often three letters -- + // REMIND's SSA, LAM, EUR. scopeGeography.v2 used to reject every + // three-letter token, so a pathway could declare `SSA` but never scope a + // value or an availability row to it. + const doc = structuredClone(v2Full) as typeof v2Full & { + geography: { regions: Record }; + }; + doc.geography.regions.SSA = ["NG", "KE", "ZA"]; + doc.keyFeatures.emissionsTrajectory.push({ + sector: "Power", + geography: "SSA", + value: "Minor decrease", + }); + doc.dataAvailability.byMetric[0].geography.push("SSA"); + const { valid, invalid } = validateDataCollect( + [{ name: "ssa.json", data: doc }], + pathwayMetadataV2, + commonSchemas, + ); + expect(invalid.flatMap((p) => p.errors)).toEqual([]); + expect(valid).toHaveLength(1); + }); + + it("routes by the document's own $schema, so a mixed corpus splits cleanly", () => { + const mixed = [v1Entry, v2Entry]; + expect( + validateDataCollect(mixed, pathwayMetadata, commonSchemas).valid.map( + (r) => r.name, + ), + ).toEqual(["v1.json"]); + expect( + validateDataCollect(mixed, pathwayMetadataV2, commonSchemas).valid.map( + (r) => r.name, + ), + ).toEqual(["v2.json"]); + }); + + it("SILENTLY DROPS documents of the other version — neither valid nor invalid", () => { + // Load-bearing behaviour, not a bug to fix here: validateDataCollect filters + // entries to the one $id it was handed. It is what lets v1 and v2 sit in + // src/data together, and it is also why pointing the loader at v2 makes every + // un-migrated v1 file disappear from the app with no error. Whatever calls + // this has to report the count it dropped, or 49 missing pathways look like a + // data bug. + const { valid, invalid } = validateDataCollect( + [v2Entry], + pathwayMetadata, + commonSchemas, + ); + expect(valid).toHaveLength(0); + expect(invalid).toHaveLength(0); + }); + + it("keeps v2's scoped keyFeatures out of v1 and vice versa", () => { + // A v1-shaped scalar is not a legal v2 value... + fail2( + { + name: "scalar-in-v2.json", + data: { ...v2Full, keyFeatures: basePathway.keyFeatures }, + }, + /keyFeatures/, + ); + // ...and a v2-shaped array is not a legal v1 value. + fail( + { + name: "array-in-v1.json", + data: { ...basePathway, keyFeatures: v2Full.keyFeatures }, + }, + /keyFeatures/, + ); + }); + + it("rejects a v2 document that still carries the removed overview fields", () => { + fail2( + { + name: "expert-overview-in-v2.json", + data: { ...v2Full, expertOverview: "Should not be here." }, + }, + /must NOT have additional properties/, + ); + }); +}); + +describe("v2 enforces its own required fields (#858)", () => { + // The v1 REQ block above deliberately still lists `expertOverview`: those cases + // validate v1 documents against v1, where it remains required. These are the v2 + // counterparts. `coreDrivers` and `dependencies` matter most — the codemod + // scaffolds them to null/[], so they are the fields most likely to be omitted + // by hand-authoring or by a partial migration of the remaining files. + const V2_REQ = [ + "id", + "name", + "description", + "publication", + "pathwayType", + "geography", + "sectors", + "pathwayDescription", + "metric", + "keyFeatures", + "coreDrivers", + "dependencies", + ]; + + for (const key of V2_REQ) { + it(`fails when required property '${key}' is missing`, () => { + const rest: Record = { ...v2Full }; + delete rest[key]; + fail2({ name: "missing.json", data: rest }, new RegExp(key)); + }); + } + + it("accepts a null pathwayDescription, but not a missing one", () => { + // Nullable-but-required: "we have no description" is authored explicitly, + // which is the same distinction #858 draws for coreDrivers and keyFeatures. + const { invalid } = validateDataCollect( + [ + { + name: "null-desc.json", + data: { ...v2Full, pathwayDescription: null }, + }, + ], + pathwayMetadataV2, + commonSchemas, + ); + expect(invalid).toHaveLength(0); + }); + + it("requires every one of the 7 coreDrivers keys, nullable though they are", () => { + const { coreDrivers } = v2Full as unknown as { + coreDrivers: Record; + }; + for (const key of Object.keys(coreDrivers)) { + const partial = { ...coreDrivers }; + delete partial[key]; + fail2( + { + name: `missing-driver-${key}.json`, + data: { ...v2Full, coreDrivers: partial }, + }, + new RegExp(key), + ); + } + }); + + it("rejects an unknown coreDrivers key", () => { + fail2( + { + name: "extra-driver.json", + data: { + ...v2Full, + coreDrivers: { + ...(v2Full as unknown as { coreDrivers: object }).coreDrivers, + madeUpDriver: "Nope.", + }, + }, + }, + /must NOT have additional properties/, + ); + }); + + it("rejects a dependency scoped to an unknown sector", () => { + fail2( + { + name: "bad-dep-sector.json", + data: { + ...v2Full, + dependencies: [ + { + dependency_name: "Technology", + dependency_description: "Needs something.", + sector: "Yak Shaving", + evidence_type: "Qualitative", + }, + ], + }, + }, + /allowed values/, + ); + }); + + it("rejects an unknown dependency_name or evidence_type", () => { + const base = { + dependency_name: "Technology", + dependency_description: "Needs something.", + sector: "Power", + evidence_type: "Qualitative", + }; + fail2( + { + name: "bad-dep-name.json", + data: { + ...v2Full, + dependencies: [{ ...base, dependency_name: "Vibes" }], + }, + }, + /allowed values/, + ); + fail2( + { + name: "bad-evidence.json", + data: { + ...v2Full, + dependencies: [{ ...base, evidence_type: "Hearsay" }], + }, + }, + /allowed values/, + ); + }); +}); diff --git a/src/utils/validateScopes.test.ts b/src/utils/validateScopes.test.ts new file mode 100644 index 00000000..dc1a9398 --- /dev/null +++ b/src/utils/validateScopes.test.ts @@ -0,0 +1,896 @@ +import { describe, it, expect } from "vitest"; +import { validateScopedEntries } from "./validateScopes"; +import type { PathwayMetadataV2 } from "../types"; +import sectorSchema from "../schema/common/sector.v1.json" with { type: "json" }; + +const SECTOR_NAMES: string[] = sectorSchema.$defs.displayName.enum; + +/** + * These cover the cross-field constraint that JSON Schema draft-07 cannot express + * (see validateScopes.ts). AJV already guarantees the shape, so the fixtures here + * only need the fields the check actually reads — hence the casts. + */ +function pathway(over: Partial): PathwayMetadataV2 { + return { + sectors: [ + { name: "Power", technologies: [] }, + { name: "Steel", technologies: [] }, + ], + geography: { + regions: { "South East Asia": ["TH", "VN"] }, + country: ["SG"], + }, + keyFeatures: {}, + dependencies: [], + ...over, + } as unknown as PathwayMetadataV2; +} + +function entry(sector: string, geography: string) { + return { sector, geography, value: "No information" }; +} + +describe("validateScopedEntries — sector axis", () => { + it("accepts a sector the pathway declares", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "South East Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("accepts the across sectors sentinel", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("across sectors", "South East Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("rejects a sector the pathway does not declare", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Cement", "South East Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/keyFeatures/emissionsTrajectory/0/sector"); + expect(errors[0]).toContain('"Cement"'); + }); + + it("accepts across sectors on a single-sector pathway (documented non-check)", () => { + const errors = validateScopedEntries( + pathway({ + sectors: [{ name: "Power", technologies: [] }], + keyFeatures: { + emissionsTrajectory: [entry("across sectors", "South East Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); +}); + +describe("validateScopedEntries — geography axis", () => { + it.each(["across regions", "South East Asia", "SG", "TH"])( + "accepts %s", + (geography) => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", geography)], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }, + ); + + it("accepts Global only when the pathway actually is global", () => { + const globalPathway = pathway({ + geography: { global: true, regions: { "South East Asia": ["TH"] } }, + }); + expect( + validateScopedEntries({ + ...globalPathway, + keyFeatures: { + emissionsTrajectory: [entry("Power", "Global")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ).toEqual([]); + }); + + it("rejects Global on a pathway that does not set geography.global", () => { + // A South-East-Asia-only pathway carrying a global-scoped value would be + // describing coverage it never claims. #858's "or the widest sentinel" + // wording allows it read literally; that defeats the point of the check. + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "Global")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"Global"'); + }); + + it("accepts a country reached only through a declared region", () => { + // The pathway declares "South East Asia": ["TH","VN"] but no standalone VN, + // so scoping to VN is *narrower* than the declaration, not outside it. + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "VN")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("rejects a mistyped region label", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "Souteast Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/keyFeatures/emissionsTrajectory/0/geography"); + expect(errors[0]).toContain('"Souteast Asia"'); + }); + + it("rejects a country the pathway does not cover", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "DE")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"DE"'); + }); + + it("accepts a declared three-letter region label", () => { + // Model regions are often three letters (REMIND's SSA, LAM, EUR), and the + // cookbook keeps a publication's label as written. + const errors = validateScopedEntries( + pathway({ + geography: { + regions: { SSA: ["NG", "KE", "ZA"] }, + } as unknown as PathwayMetadataV2["geography"], + keyFeatures: { + emissionsTrajectory: [entry("Power", "SSA")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("still rejects an ISO alpha-3 code the pathway does not declare", () => { + // This check, not a schema-level ban on three-letter tokens, is what stops + // `THA` being written for `TH`: only declared labels, their alpha-2 + // members, declared countries and the sentinels are allowed. + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "THA")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"THA"'); + expect(errors[0]).toContain("is not a geography this pathway declares"); + }); +}); + +describe("validateScopedEntries — reporting", () => { + it("reports both axes of a single bad entry, and indexes each entry", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [ + entry("Power", "South East Asia"), + entry("Cement", "Narnia"), + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(2); + expect(errors.every((e) => e.includes("/emissionsTrajectory/1/"))).toBe( + true, + ); + }); + + it("checks every keyFeatures field, not just the first", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [entry("Power", "South East Asia")], + policyAmbition: [entry("Cement", "South East Asia")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/keyFeatures/policyAmbition/0/sector"); + }); + + it("passes an empty entries array — absent at every scope is legal", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); +}); + +describe("validateScopedEntries — one value per scope", () => { + const dup = (geography: string, value: string) => ({ + sector: "Power", + geography, + value, + }); + + it("rejects two entries at the same scope with different values", () => { + // uniqueItems compares whole entries, so these are "unique" to the schema. + // Left unchecked, the resolver displays one while search matches both. + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [ + dup("South East Asia", "Significant decrease"), + dup("South East Asia", "Minor increase"), + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/keyFeatures/emissionsTrajectory/1"); + expect(errors[0]).toContain("duplicates the scope of"); + // Names the entry it collides with, so the fix is obvious in a long list. + expect(errors[0]).toContain("/keyFeatures/emissionsTrajectory/0"); + }); + + it("accepts the same value at genuinely different scopes", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [ + dup("South East Asia", "Significant decrease"), + dup("SG", "Significant decrease"), + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("distinguishes scopes that differ only by sector", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [ + { sector: "Power", geography: "SG", value: "Minor increase" }, + { sector: "Steel", geography: "SG", value: "Minor decrease" }, + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("reports every repeat, not just the second", () => { + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [ + dup("SG", "Significant decrease"), + dup("SG", "Minor increase"), + dup("SG", "Low or no change"), + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toHaveLength(2); + // Both point back at index 0 rather than chaining 1->2. + expect(errors.every((e) => e.includes("emissionsTrajectory/0"))).toBe(true); + }); + + it("scopes the check per field, not across the whole object", () => { + // The same (sector, geography) in two different fields is normal. + const errors = validateScopedEntries( + pathway({ + keyFeatures: { + emissionsTrajectory: [dup("SG", "Significant decrease")], + policyAmbition: [dup("SG", "High ambition policies")], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); + + it("does not confuse scopes whose parts concatenate alike", () => { + // Guards the composite key: "A" + "BC" must not collide with "AB" + "C". + const errors = validateScopedEntries( + pathway({ + sectors: [{ name: "Power", technologies: [] }], + geography: { regions: { "Power SG": ["SG"] }, country: ["SG"] }, + keyFeatures: { + emissionsTrajectory: [ + { sector: "Power", geography: "SG", value: "Minor increase" }, + { sector: "Power", geography: "Power SG", value: "Minor decrease" }, + ], + } as unknown as PathwayMetadataV2["keyFeatures"], + }), + ); + expect(errors).toEqual([]); + }); +}); + +describe("validateScopedEntries — dependencies", () => { + it("accepts a declared sector", () => { + const errors = validateScopedEntries( + pathway({ + dependencies: [ + { + dependency_name: "Technology", + dependency_description: "Needs grid upgrades.", + sector: "Power", + evidence_type: "Qualitative", + }, + ] as unknown as PathwayMetadataV2["dependencies"], + }), + ); + expect(errors).toEqual([]); + }); + + it("rejects an undeclared sector", () => { + const errors = validateScopedEntries( + pathway({ + dependencies: [ + { + dependency_name: "Technology", + dependency_description: "Needs grid upgrades.", + sector: "Aviation", + evidence_type: "Qualitative", + }, + ] as unknown as PathwayMetadataV2["dependencies"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/dependencies/0/sector"); + }); + + it("rejects the across sectors sentinel, which is not legal here", () => { + const errors = validateScopedEntries( + pathway({ + dependencies: [ + { + dependency_name: "Technology", + dependency_description: "Needs grid upgrades.", + sector: "across sectors", + evidence_type: "Qualitative", + }, + ] as unknown as PathwayMetadataV2["dependencies"], + }), + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"across sectors"'); + }); +}); + +describe("validateScopedEntries — technologies belong to their sector (#461)", () => { + const withSectors = (sectors: unknown) => + validateScopedEntries( + pathway({ sectors: sectors as PathwayMetadataV2["sectors"] }), + ); + + it("accepts technologies the sector lists", () => { + expect( + withSectors([{ name: "Power", technologies: ["Solar", "Wind", "Coal"] }]), + ).toEqual([]); + }); + + it("accepts every technology Power defines", () => { + expect( + withSectors([ + { + name: "Power", + technologies: [ + "Biomass", + "Coal", + "Gas", + "Hydro", + "Nuclear", + "Oil", + "Other", + "Renewables", + "Solar", + "Wind", + ], + }, + ]), + ).toEqual([]); + }); + + it("rejects a technology outside its sector's list, naming the value", () => { + const errors = withSectors([ + { name: "Power", technologies: ["Solar", "Hydrogen Use"] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectors/0/technologies/1"); + expect(errors[0]).toContain('"Hydrogen Use"'); + expect(errors[0]).toContain('"Power"'); + }); + + it("accepts an empty list for every sector the schema allows", () => { + // The legal state for the 14 sectors whose technologies are not defined yet, + // and what all four TransitionZero pathways carry today. + for (const name of SECTOR_NAMES) { + expect(withSectors([{ name, technologies: [] }])).toEqual([]); + } + }); + + it("rejects a non-empty list on a sector with no definition", () => { + // Closed by default. Passing this through would mean the next data round + // populates a sector's technologies and nothing checks them. + const errors = withSectors([ + { name: "Steel", technologies: ["Hydrogen Use"] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectors/0/technologies"); + expect(errors[0]).toContain('"Steel"'); + }); + + it("names the fix, since the offending data may well be correct", () => { + // The likeliest cause is a sector whose taxonomy nobody has written down + // yet, so the message has to say where to write it. + const errors = withSectors([ + { name: "Cement", technologies: ["Carbon Capture and Storage"] }, + ]); + expect(errors[0]).toContain("SECTORS_BY_KEY"); + expect(errors[0]).toContain("timeseriesTaxonomy.ts"); + expect(errors[0]).toContain('"Carbon Capture and Storage"'); + }); + + it("reports one error per undefined sector, not one per technology", () => { + // The fix is a single edit -- define the sector -- so listing its + // technologies individually would be noise. + const errors = withSectors([ + { name: "Steel", technologies: ["Hydrogen Use", "Electrification"] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"Electrification"'); + expect(errors[0]).toContain('"Hydrogen Use"'); + }); + + it("indexes each sector, and checks them all", () => { + const errors = withSectors([ + { name: "Power", technologies: ["Solar"] }, + { name: "Steel", technologies: ["Hydrogen Use"] }, + { name: "Cement", technologies: [] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectors/1/technologies"); + }); + + it("passes a pathway with no sectors at all", () => { + // AJV does not require `sectors`, so the check must not assume it is there. + expect(withSectors(undefined)).toEqual([]); + }); + + it("accepts pathway-level segments the sector lists", () => { + expect( + withSectors([ + { + name: "Power", + technologies: [], + segments: ["Power generation", "Transmission and distribution"], + }, + ]), + ).toEqual([]); + }); + + it("rejects a pathway-level segment belonging to another sector", () => { + const errors = withSectors([ + { name: "Power", technologies: [], segments: ["Ironmaking"] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectors/0/segments/0"); + expect(errors[0]).toContain('"Ironmaking"'); + expect(errors[0]).toContain("is not a segment of sector"); + }); + + it.each(["Steel", "Aviation"])( + "checks segments under %s, which has segments but no technology list", + (name) => { + // The case the Power test above cannot reach: the technology check used + // to `return` for a sector without a technology vocabulary, and that + // skipped the segment check entirely -- for exactly these two sectors. + const errors = withSectors([ + { name, technologies: [], segments: ["Power generation"] }, + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectors/0/segments/0"); + expect(errors[0]).toContain('"Power generation"'); + }, + ); + + it("accepts an absent segments list — the field is optional", () => { + expect(withSectors([{ name: "Power", technologies: [] }])).toEqual([]); + }); + + it("passes segments under a sector whose segments are undefined", () => { + // Unlike `technologies`, which rejects this: `segments` is optional, so + // there is no authored `[]` to fall back to, and rejecting would make the + // field unusable for the twelve sectors the cookbook has not reached. + expect( + withSectors([ + { name: "Cement", technologies: [], segments: ["Power generation"] }, + ]), + ).toEqual([]); + }); +}); + +describe("validateScopedEntries — dataAvailability rows (#870)", () => { + /** A row that passes every check, for tests to vary one field of. */ + const ROW = { + metricName: "Capacity", + sector: "Power", + sectorSegment: ["Power generation"], + geography: ["South East Asia"], + timeResolution: "1-year steps", + dataFormat: "Tabular", + granularity: ["Solar"], + scopeLimitations: "Excludes off-grid generation.", + }; + + const withRows = (rows: unknown[], over: Record = {}) => + validateScopedEntries( + pathway({ + metric: ["Capacity", "Generation"], + dataAvailability: { overall: null, byMetric: rows }, + ...over, + } as unknown as Partial), + ); + + const row = (over: Record = {}) => ({ ...ROW, ...over }); + + it("accepts a row that satisfies every check", () => { + expect(withRows([row()])).toEqual([]); + }); + + it("passes a pathway with no dataAvailability at all", () => { + // Optional field: authoring is incremental, so absence is not a defect. + expect(validateScopedEntries(pathway({}))).toEqual([]); + }); + + it("passes an empty byMetric array", () => { + expect(withRows([])).toEqual([]); + }); + + it("rejects a sector the pathway does not declare", () => { + const errors = withRows([row({ sector: "Cement" })]); + expect( + errors.some((e) => e.includes("/sector") && e.includes('"Cement"')), + ).toBe(true); + }); + + it("rejects a geography the pathway does not cover", () => { + const errors = withRows([row({ geography: ["Narnia"] })]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/dataAvailability/byMetric/0/geography/0"); + expect(errors[0]).toContain('"Narnia"'); + }); + + it("checks every member of the geography list, and indexes them", () => { + // The cookbook types Geography coverage as Multiple, so one bad token among + // good ones must still be caught — and named by position. + const errors = withRows([ + row({ geography: ["South East Asia", "Narnia", "SG"] }), + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/geography/1"); + expect(errors[0]).toContain('"Narnia"'); + }); + + it("accepts a geography list mixing global, region and country scope", () => { + // 'Global; Western Europe: [...]; JP: [JP]' in the cookbook's notation: one + // metric can be projected at several levels at once. Global is legal only + // because this pathway declares it — allowedGeographies gates it on + // geography.global, exactly as it does for a keyFeatures entry. + expect( + withRows([row({ geography: ["Global", "South East Asia", "SG"] })], { + geography: { + global: true, + regions: { "South East Asia": ["ID", "TH", "VN"] }, + country: ["SG"], + }, + }), + ).toEqual([]); + }); + + it("accepts a metric the pathway's own `metric` list does not name", () => { + // Deliberate: `metric` and the availability row key are two different + // cookbook variables (register item D16), and a covered sector earns a row + // for every metric in the extended list -- the unreported ones marked + // `Not covered`. The old rule required the row's metric to appear in + // `metric`, which made exactly those rows unauthorable. + expect( + withRows([row({ metricName: "Transmission lines" })], { + metric: ["Capacity"], + }), + ).toEqual([]); + }); + + it("rejects a metric that is not the sector's, naming the value", () => { + // Reached directly here: in production AJV rejects a non-enum metricName + // first, and Power currently defines all five members of that enum, so this + // path only becomes live for real data once a second sector defines metrics. + const errors = withRows([row({ metricName: "Water Use" })], { + metric: ["Water Use"], + }); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("is not a metric of sector"); + expect(errors[0]).toContain('"Water Use"'); + }); + + it("accepts a metric under a sector whose metrics are undefined", () => { + // Cement has no availability-metric list. Rejecting here would make + // dataAvailability unauthorable for the twelve sectors the cookbook has not + // reached, which is the blockage `UNSEGMENTED` exists to avoid. + expect( + withRows( + [ + row({ + sector: "Cement", + sectorSegment: ["No information"], + granularity: ["Unspecified"], + }), + ], + { + sectors: [ + { name: "Power", technologies: [] }, + { name: "Cement", technologies: [] }, + ], + }, + ), + ).toEqual([]); + }); + + it("rejects a named segment under a sector with no segments defined", () => { + const errors = withRows( + [ + row({ + sector: "Cement", + sectorSegment: ["Energy storage"], + granularity: ["Unspecified"], + }), + ], + { + sectors: [ + { name: "Power", technologies: [] }, + { name: "Cement", technologies: [] }, + ], + }, + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/sectorSegment"); + expect(errors[0]).toContain('"Energy storage"'); + // Names the one segment that is legal, and how to define the rest. + expect(errors[0]).toContain('"No information"'); + expect(errors[0]).toContain("SECTORS_BY_KEY"); + }); + + it("accepts the No information segment under any sector", () => { + expect(withRows([row({ sectorSegment: ["No information"] })])).toEqual([]); + }); + + it("rejects granularity that is not a technology of the sector", () => { + const errors = withRows([row({ granularity: ["Solar", "Hydrogen Use"] })]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/granularity/1"); + expect(errors[0]).toContain('"Hydrogen Use"'); + }); + + it("rejects granularity on a sector with no technologies defined", () => { + const errors = withRows( + [ + row({ + sector: "Cement", + sectorSegment: ["No information"], + granularity: ["Solar"], + }), + ], + { + sectors: [ + { name: "Power", technologies: [] }, + { name: "Cement", technologies: [] }, + ], + }, + ); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("no technologies are defined"); + }); + + it("accepts an emissions scope as granularity", () => { + // Not a technology: emissions metrics break down by scope, so the + // granularityBreakdown vocabulary is exempt from the technology check. + expect(withRows([row({ granularity: ["Scope 1 & 2"] })])).toEqual([]); + }); + + it("accepts Unspecified granularity — the pathway need not say", () => { + expect(withRows([row({ granularity: ["Unspecified"] })])).toEqual([]); + }); + + it.each(["geography", "granularity"])( + "rejects a sentinel sharing the %s list with a real value", + (field) => { + // A sentinel stands in for the whole list. "Not covered" beside a real + // breakdown would claim the pair is both uncovered and broken down. + const over = + field === "geography" + ? { geography: ["South East Asia", "Unspecified"] } + : { granularity: ["Solar", "Unspecified"] }; + const errors = withRows([row(over)]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain(`/${field}`); + expect(errors[0]).toContain("must be its only member"); + }, + ); + + it.each([ + [ + "sectorSegment", + { sectorSegment: ["No information", "Power generation"] }, + ], + ["granularity", { granularity: ["No information", "Solar"] }], + ])( + 'rejects "No information" sharing the %s list with a real value', + (field, over) => { + // The third absence value. It is legal on these two fields, so it was + // checked for membership -- but it is not in SENTINELS, so the + // must-stand-alone rule never saw it. + const errors = withRows([row(over)]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain(`/${field}`); + expect(errors[0]).toContain('"No information"'); + expect(errors[0]).toContain("must be its only member"); + }, + ); + + it('still rejects "No information" as a geography token', () => { + // Why it is not simply added to SENTINELS: that list also exempts a token + // from the geography membership check. + const errors = withRows([row({ geography: ["No information"] })]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("is not a geography this pathway declares"); + }); + + it("accepts a row that is Not covered across every variable", () => { + // The cookbook requires a row for each allowable (sector, metric) pair even + // when the pair is uncovered, so this is a normal authored row. + expect( + withRows([ + row({ + sectorSegment: ["Not covered"], + geography: ["Not covered"], + timeResolution: "Not covered", + dataFormat: "Not covered", + granularity: ["Not covered"], + scopeLimitations: "Not covered", + }), + ]), + ).toEqual([]); + }); + + it("rejects Not covered mixed with authored values on one row", () => { + // "Not covered" is a property of the (sector, metric) pair, so it cannot + // apply to one variable and not another. + const errors = withRows([row({ dataFormat: "Not covered" })]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('mixes "Not covered"'); + expect(errors[0]).toContain('"Unspecified"'); + }); + + it("accepts Unspecified across several variables at once", () => { + // Unlike "Not covered", Unspecified is per-variable: a covered pair may be + // documented for some variables and silent on others. + expect( + withRows([ + row({ + timeResolution: "Unspecified", + granularity: ["Unspecified"], + scopeLimitations: "Unspecified", + }), + ]), + ).toEqual([]); + }); + + it("rejects two rows at the same scope", () => { + // uniqueItems permits these: they agree on the scope and differ elsewhere, + // so the schema sees two distinct entries. The table has one cell to render + // them in. + const errors = withRows([ + row(), + row({ timeResolution: "5-year steps", granularity: ["Unspecified"] }), + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/dataAvailability/byMetric/1"); + expect(errors[0]).toContain("duplicates the scope of"); + expect(errors[0]).toContain("/dataAvailability/byMetric/0"); + }); + + it("treats two rows covering the same places as one scope, in any order", () => { + // The key is the geography set, not the list: order is an authoring + // accident, so reordering must not smuggle a duplicate row past the check. + const errors = withRows([ + row({ geography: ["South East Asia", "SG"] }), + row({ geography: ["SG", "South East Asia"], dataFormat: "Text" }), + ]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("duplicates the scope of"); + }); + + it.each([ + ["metric", { metricName: "Generation" }], + ["segment", { sectorSegment: ["Energy storage"] }], + ["geography", { geography: ["SG"] }], + [ + "sector", + { + sector: "Steel", + // Steel's own availability-metric list, not Power's -- the two sectors + // legitimately name different metrics. + metricName: "Scrap share", + sectorSegment: ["No information"], + granularity: ["Unspecified"], + }, + ], + ])("treats rows differing only by %s as distinct scopes", (_axis, over) => { + expect(withRows([row(), row(over)])).toEqual([]); + }); + + it("does not confuse scopes whose parts concatenate alike", () => { + // Guards the NUL-joined key the same way the keyFeatures test does. + expect( + withRows([ + row({ sectorSegment: ["Power generation"], geography: ["SG"] }), + row({ sectorSegment: ["No information"], geography: ["SG"] }), + ]), + ).toEqual([]); + }); + + it("indexes each row, and checks them all", () => { + const errors = withRows([row(), row({ geography: ["Narnia"] })]); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("/byMetric/1/geography"); + }); + + it("reports every failing check on one row", () => { + const errors = withRows([ + row({ + sector: "Cement", + geography: ["Narnia"], + dataFormat: "Not covered", + }), + ]); + // undeclared sector, unresolvable geography, granularity under an undefined + // sector, and "Not covered" mixed with authored values. + expect(errors.length).toBeGreaterThanOrEqual(4); + expect(errors.every((e) => e.includes("/byMetric/0"))).toBe(true); + }); +}); diff --git a/src/utils/validateScopes.ts b/src/utils/validateScopes.ts new file mode 100644 index 00000000..7a5a9929 --- /dev/null +++ b/src/utils/validateScopes.ts @@ -0,0 +1,506 @@ +/** + * Structural checks for v2 scoped keyFeatures entries (#858). + * + * #858 requires an entry's `sector`/`geography` to be declared in the pathway's + * own `sectors`/`geography`, or be the widest sentinel. That constraint spans + * sibling data with dynamic keys (`geography.regions` is an open object of + * author-defined labels), which JSON Schema draft-07 cannot express: there is no + * way to point an `enum` at another part of the same document, and AJV's `$data` + * can only reference a single value, not compute the union of region labels, + * region members, and country codes that this needs. + * + * So `scopeGeography.v2.json` validates the *shape* — a non-blank, non-3-letter + * string — and this module validates the *reference*. Without it a mistyped + * region label ("Souteast Asia") would validate cleanly and then silently match + * nothing at search time, which is the worst of both worlds. + * + * It also enforces one-value-per-scope, which `uniqueItems` cannot: that keyword + * compares whole entries, so two at the same (sector, geography) with different + * values are "unique" to the schema while being contradictory as data. + * + * It also enforces #461 -- that a technology belongs to the sector it is attached + * to. #461 suggests mirroring the `if`/`then` sector conditional in + * `pathwayTimeseries.v1.json`, but that keyword pair defeats + * `json-schema-to-typescript`: the timeseries `data` items use exactly that shape + * and generate as `{ [k: string]: unknown }[]`. Applying it to `sectors.items` + * would collapse today's `{ name; technologies }` object type and break every + * consumer of `Sector`, so the constraint lives here with its siblings instead. + * + * #870's `dataAvailability` rows are checked here for the same reasons: their + * `metricName`/`sectorSegment`/`granularity` vocabularies are sector-conditional, + * their scope must resolve against the pathway's own coverage, and the two + * sentinel rules span sibling properties -- a sentinel must be its list's only + * member, and `Not covered` describes the whole row rather than one variable. + * None of that is expressible in draft-07. + * + * Run from `scripts/schema-check-files.ts`, so `npm run schema:check` gates all + * of it. + * + * Errors are formatted like AJV's (` `) so callers can + * merge them into the same `ValidationProblem.errors` list without special-casing. + */ +import type { PathwayMetadataV2 } from "../types/pathwayMetadata.v2"; +import { dataAvailabilitySchema } from "../schema/common/index.ts"; +import { + availabilityMetricBelongsToSector, + availabilityMetricsForSector, + segmentBelongsToSector, + segmentsForSector, + technologiesForSector, + technologyBelongsToSector, + UNSEGMENTED, +} from "./timeseriesTaxonomy.ts"; + +/** `$id` of the schema these checks apply to. */ +export const PATHWAY_METADATA_V2_ID = + "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +/** Sector sentinel meaning "the union of this pathway's own declared sectors". */ +export const ACROSS_SECTORS = "across sectors"; + +/** Geography sentinels: widest possible, and a multi-region non-global aggregate. */ +export const GLOBAL_SCOPE = "Global"; +export const ACROSS_REGIONS = "across regions"; + +/** + * The two authored-absence values every dataAvailability variable shares + * (cookbook tpr_cookbook_20260924). + * + * They are not interchangeable and neither is a null. `Unspecified` says the + * pathway covers this (sector, metric) pair but does not state the value; + * `Not covered` says it does not cover the pair at all, and so applies to every + * variable on the row at once. Keeping both explicit is the same rule #858 sets + * for keyFeatures, where an authored "No information" terminates the fallback + * chain and an absent entry does not. + */ +export const UNSPECIFIED = "Unspecified"; +export const NOT_COVERED = "Not covered"; + +const SENTINELS: readonly string[] = [UNSPECIFIED, NOT_COVERED]; + +function isSentinel(value: string): boolean { + return SENTINELS.includes(value); +} + +/** + * Granularity members drawn from the `granularityBreakdown` vocabulary rather + * than from the sector's technologies — emissions scopes and the sentinels. + * They are exempt from the technology check below, not from the schema enum. + * + * Read off the schema rather than restated here, the same way `filterRegions` + * derives `ALL_COUNTRY_CODES` from `countryCode.v1`: the two lists must agree, + * and a copy is a copy that drifts. + */ +const GRANULARITY_BREAKDOWN: ReadonlySet = new Set( + (dataAvailabilitySchema.$defs as Record) + ?.granularityBreakdown?.enum ?? [], +); + +function isBreakdownValue(value: string): boolean { + return GRANULARITY_BREAKDOWN.has(value); +} + +/** + * Whether each of a dataAvailability row's six variables reads `Not covered`. + * + * One entry per variable rather than a flat list of values, because the rule it + * feeds is about agreement *between* variables: either they all say the pair is + * uncovered or none of them do. `dataFormat` counts even though its enum has no + * `Unspecified` — it does have `Not covered`. + */ +function notCoveredByVariable(row: { + geography: readonly string[]; + granularity: readonly string[]; + sectorSegment: readonly string[]; + timeResolution: string; + dataFormat: string; + scopeLimitations: string; +}): boolean[] { + return [ + row.geography.includes(NOT_COVERED), + row.granularity.includes(NOT_COVERED), + row.sectorSegment.includes(NOT_COVERED), + row.timeResolution === NOT_COVERED, + row.dataFormat === NOT_COVERED, + row.scopeLimitations === NOT_COVERED, + ]; +} + +type ScopedEntry = { sector: string; geography: string; value: unknown }; + +/** + * Every geography token an entry on this pathway may legitimately name: each + * declared region label, every country inside those regions, every standalone + * country, and the sentinels where they apply. Region members count because a + * pathway that covers "South East Asia" does cover Thailand — scoping an entry + * to `TH` is more specific than the pathway's own declaration, not outside it. + * + * `Global` is allowed only when the pathway actually sets `geography.global`. + * #858 phrases the rule as "declared by the pathway, or the widest sentinel", + * which read literally would let a South-East-Asia-only pathway carry a + * global-scoped value — describing coverage it never claims, and defeating the + * point of the check. `across regions` stays unconditional: #858 reserves it for a + * multi-region non-global aggregate without saying when it applies, and no file + * in the corpus uses it yet, so gating it would be inventing a rule. + */ +function allowedGeographies(pathway: PathwayMetadataV2): Set { + const allowed = new Set([ACROSS_REGIONS]); + const geo = pathway.geography; + if (!geo || typeof geo !== "object") return allowed; + if (geo.global === true) allowed.add(GLOBAL_SCOPE); + if (geo.regions) { + for (const [label, members] of Object.entries(geo.regions)) { + allowed.add(label); + if (Array.isArray(members)) members.forEach((m) => allowed.add(m)); + } + } + if (Array.isArray(geo.country)) geo.country.forEach((c) => allowed.add(c)); + return allowed; +} + +/** + * Sectors an entry may name: the pathway's own, plus the `across sectors` + * sentinel. + * + * Note what is deliberately *not* checked: #858 remarks that `across sectors` is + * "only meaningful for multi-sector pathways", but a single-sector pathway using + * it is harmless — it resolves to that one sector — so flagging it would be a + * false positive on a legal document rather than a caught mistake. + */ +function declaredSectors(pathway: PathwayMetadataV2): Set { + const declared = new Set(); + for (const s of pathway.sectors ?? []) { + if (s?.name) declared.add(s.name); + } + return declared; +} + +/** + * The trailing clause of a "not a valid X of sector Y" message. + * + * Three cases, because they call for three different actions. Undefined means + * nobody has written that sector's vocabulary down, and the fix is to write it. + * Defined-but-empty means someone has, and the answer is genuinely "none" -- so + * saying "(allowed: )" would read as a bug in the checker rather than an answer. + * Only a non-empty list can usefully be listed. + */ +function allowedClause( + defined: readonly string[] | undefined, + axis: string, +): string { + if (!defined) { + return ( + `-- no ${axis} are defined for that sector. Add them to SECTORS_BY_KEY in` + + ` src/utils/timeseriesTaxonomy.ts.` + ); + } + if (defined.length === 0) { + return `-- that sector defines no ${axis}.`; + } + return `(allowed: ${quote(defined)})`; +} + +function quote(values: Iterable): string { + return [...values] + .sort((a, b) => a.localeCompare(b)) + .map((v) => `"${v}"`) + .join(", "); +} + +/** + * Check one v2 metadata document's scope references. Returns an empty array when + * everything resolves. Assumes the document already passed AJV against + * `pathwayMetadata.v2.json`, so shapes are trusted and only references are tested. + */ +export function validateScopedEntries(pathway: PathwayMetadataV2): string[] { + const errors: string[] = []; + const declared = declaredSectors(pathway); + const entrySectors = new Set([ACROSS_SECTORS, ...declared]); + const geographies = allowedGeographies(pathway); + + const keyFeatures = (pathway.keyFeatures ?? {}) as Record< + string, + ScopedEntry[] | undefined + >; + for (const [field, entries] of Object.entries(keyFeatures)) { + if (!Array.isArray(entries)) continue; + // Tracks the first index each (sector, geography) pair was seen at, so a + // repeat can name its twin. See the duplicate check below for why. + const seenScopes = new Map(); + entries.forEach((entry, i) => { + const at = `/keyFeatures/${field}/${i}`; + if (!entrySectors.has(entry.sector)) { + errors.push( + `${at}/sector "${entry.sector}" is not a sector this pathway declares` + + ` (allowed: ${quote(entrySectors)})`, + ); + } + if (!geographies.has(entry.geography)) { + errors.push( + `${at}/geography "${entry.geography}" is not a geography this pathway` + + ` declares (allowed: ${quote(geographies)})`, + ); + } + + // One scope, one value. The schema's `uniqueItems` only rejects entries + // that are identical including their value, so two entries at the same + // (sector, geography) carrying *different* values validate cleanly -- and + // then disagree downstream: the resolver picks one to display while search + // matches the field under both, so a pathway surfaces under a value its + // own detail page does not show. The likely author intent is an override, + // which is not what the data expresses, so reject it rather than pick a + // winner by document order. + const scope = `${entry.sector}\u0000${entry.geography}`; + const firstSeen = seenScopes.get(scope); + if (firstSeen === undefined) { + seenScopes.set(scope, i); + } else { + errors.push( + `${at} duplicates the scope of /keyFeatures/${field}/${firstSeen}` + + ` (sector "${entry.sector}", geography "${entry.geography}").` + + ` Each scope may carry only one value; to vary a value, vary the scope.`, + ); + } + }); + } + + // #461: a technology must belong to the sector it is attached to. The schema + // types `technologies` as the flat 31-member enum, which is why the generated + // type still reads as "a bit of a random list" -- that breadth is answered here + // rather than in the type. + // + // Closed by default: a sector whose technologies are not defined in + // `timeseriesTaxonomy.ts` accepts only an empty list. The alternative -- passing + // anything through for undefined sectors -- means the next data round populates + // technologies for a new sector and nothing checks them, which is the failure + // this whole check exists to prevent. Rejecting instead makes the missing + // definition impossible to miss, and the message says exactly where to add it. + (pathway.sectors ?? []).forEach((sector, i) => { + if (!sector?.name) return; + // An if/else rather than an early return: the segments check below must run + // for every sector, and the sectors with no technology list -- Steel and + // Aviation -- are exactly the ones that do have segments. A `return` here + // once skipped them, so a Steel entry naming a Power segment validated. + const allowed = technologiesForSector(sector.name); + const technologies = sector.technologies ?? []; + if (!allowed) { + if (technologies.length > 0) { + errors.push( + `/sectors/${i}/technologies lists ${quote(technologies)} but no` + + ` technology list is defined for sector "${sector.name}". Add one to` + + ` SECTORS_BY_KEY in src/utils/timeseriesTaxonomy.ts, or use [].`, + ); + } + } else { + technologies.forEach((technology, t) => { + if (technologyBelongsToSector(technology, sector.name) !== "yes") { + errors.push( + `/sectors/${i}/technologies/${t} "${technology}" is not a technology of` + + ` sector "${sector.name}" ` + + allowedClause(allowed, "technologies"), + ); + } + }); + } + + /* + The pathway-level `segments` list, same rule as `technologies` above and + for the same reason -- but without the undefined-sector rejection. + + `technologies` rejects a non-empty list under a sector with no definition, + because the field is required and `[]` is the authored way to say "none". + `segments` is optional, so there is no `[]` to fall back to: rejecting + would make the field unusable for the twelve sectors whose segments the + cookbook has not written down, rather than catching a mistake. An + undefined sector therefore passes, as it does for a dataAvailability row. + */ + const segmentsAllowed = segmentsForSector(sector.name); + if (segmentsAllowed) { + (sector.segments ?? []).forEach((segment, g) => { + if (segmentBelongsToSector(segment, sector.name) !== "yes") { + errors.push( + `/sectors/${i}/segments/${g} "${segment}" is not a segment of` + + ` sector "${sector.name}" ` + + allowedClause(segmentsAllowed, "segments"), + ); + } + }); + } + }); + + // #870: dataAvailability rows. Optional -- authoring is incremental, and a + // pathway without the field is not an invalid pathway. + const availability = pathway.dataAvailability; + if (availability && Array.isArray(availability.byMetric)) { + // First index each (metricName, sector, sectorSegment, geography) was seen + // at. NUL-joined so no combination of parts can collide with another; see + // the keyFeatures duplicate check above for the same reasoning. + const seenRows = new Map(); + + availability.byMetric.forEach((row, i) => { + const at = `/dataAvailability/byMetric/${i}`; + + if (!declared.has(row.sector)) { + errors.push( + `${at}/sector "${row.sector}" is not a sector this pathway declares` + + ` (allowed: ${quote(declared)})`, + ); + } + // The cookbook types Geography coverage as "a subset of the geographies + // listed at pathway level", so every member is checked the way the single + // token used to be. A sentinel stands in for the whole list rather than + // naming a place, so it is legal only alone -- checked with the other + // sentinels below. + row.geography.forEach((token, g) => { + if (isSentinel(token)) return; + if (!geographies.has(token)) { + errors.push( + `${at}/geography/${g} "${token}" is not a geography this pathway` + + ` declares (allowed: ${quote(geographies)})`, + ); + } + }); + + /* + Checked against the sector's *availability* metric vocabulary, not the + pathway's own `metric` array. + + The pathway-level check this replaces required every row to name a + metric the pathway lists in `metric`. Under the cookbook that is wrong + twice over: the two metric variables are separate vocabularies (register + item D16), and a covered sector earns a row for *every* metric in the + extended list, with the ones it does not report marked `Not covered`. So + the old rule made the cookbook's completeness requirement unsatisfiable + -- a `Not covered` row was rejected precisely because it was not + reported. + + This closes the axis by default, unlike the comment that used to sit + here: `availabilityMetricsForSector` is defined for all three sectors the + cookbook covers, and a sector with no definition still answers + "unknown" and passes. + */ + if ( + availabilityMetricBelongsToSector(row.metricName, row.sector) === "no" + ) { + errors.push( + `${at}/metricName "${row.metricName}" is not a metric of sector` + + ` "${row.sector}" ` + + allowedClause(availabilityMetricsForSector(row.sector), "metrics"), + ); + } + + // Multi-valued since the cookbook types it Multiple, so each member is + // checked the way the single value used to be. The sentinels are legal + // under every sector, like UNSEGMENTED; that they must then be the list's + // only member is checked with the other sentinels below. + row.sectorSegment.forEach((segment, g) => { + if (isSentinel(segment)) return; + if (segmentBelongsToSector(segment, row.sector) === "yes") return; + const defined = segmentsForSector(row.sector); + errors.push( + `${at}/sectorSegment/${g} "${segment}" is not a segment of sector` + + ` "${row.sector}" ` + + // UNSEGMENTED and the two sentinels are always legal, so they + // belong in every allowed list. + allowedClause( + [...(defined ?? []), UNSEGMENTED, ...SENTINELS], + "segments", + ) + + (defined + ? "" + : ` No segments are defined for that sector; add them to` + + ` SECTORS_BY_KEY in src/utils/timeseriesTaxonomy.ts.`), + ); + }); + + // Same rule as sectors[].technologies (#461): a breakdown dimension has to + // be a technology the sector actually has. Members of the + // granularityBreakdown vocabulary are not technologies and are skipped -- + // the schema enum is what constrains those. + row.granularity.forEach((value, g) => { + if (isBreakdownValue(value)) return; + if (technologyBelongsToSector(value, row.sector) !== "yes") { + errors.push( + `${at}/granularity/${g} "${value}" is not a technology of` + + ` sector "${row.sector}" ` + + allowedClause(technologiesForSector(row.sector), "technologies"), + ); + } + }); + + // A sentinel replaces the list rather than joining it: "Not covered" + // alongside a real breakdown would say both that the pair is uncovered and + // how it is broken down. + // + // `No information` (UNSEGMENTED) is an absence too, and must also stand + // alone -- but only on the two fields where it is a legal member. It is + // deliberately not in SENTINELS: that list also exempts a token from the + // geography membership check, which would make it a legal geography. + for (const [field, values, absences] of [ + ["geography", row.geography, SENTINELS], + ["granularity", row.granularity, [...SENTINELS, UNSEGMENTED]], + ["sectorSegment", row.sectorSegment, [...SENTINELS, UNSEGMENTED]], + ] as const) { + const sentinel = values.findIndex((v) => absences.includes(v)); + if (sentinel !== -1 && values.length > 1) { + errors.push( + `${at}/${field} lists "${values[sentinel]}" alongside` + + ` ${values.length - 1} other value(s). A sentinel stands in for the` + + ` whole list, so it must be its only member.`, + ); + } + } + + // "Not covered" is a property of the (sector, metric) pair, not of one + // variable: the cookbook requires a row for every allowable pair and marks + // an uncovered pair across the whole row. A row uncovered for one variable + // and authored for another is a half-filled row, not data. + const notCovered = notCoveredByVariable(row); + if (notCovered.some(Boolean) && !notCovered.every(Boolean)) { + errors.push( + `${at} mixes "${NOT_COVERED}" with other values. "${NOT_COVERED}"` + + ` describes the whole (sector, metric) pair, so when it applies every` + + ` variable on the row carries it. Use "${UNSPECIFIED}" for a covered` + + ` pair the pathway says nothing about.`, + ); + } + + const scope = [ + row.metricName, + row.sector, + // Order within either list is an authoring accident, not data, so the + // key is the set: two rows covering the same segments and places + // collide however they happen to be written down. + [...row.sectorSegment].sort().join(","), + [...row.geography].sort().join(","), + ].join("\u0000"); + const firstSeen = seenRows.get(scope); + if (firstSeen === undefined) { + seenRows.set(scope, i); + } else { + errors.push( + `${at} duplicates the scope of /dataAvailability/byMetric/${firstSeen}` + + ` (metric "${row.metricName}", sector "${row.sector}", segments` + + ` ${quote(row.sectorSegment)}, geography ${quote(row.geography)}).` + + ` Each combination may describe only one row; the table has one cell` + + ` per column to render it in.`, + ); + } + }); + } + + // dependencies are descriptive and not part of the inheritance chain, but + // #858 still scopes each to a sector, and that sector must be a real one. + // Note this uses `declared`, not `entrySectors`: the schema types this field as + // the plain sector enum, so `across sectors` is not a legal value here. + (pathway.dependencies ?? []).forEach((dep, i) => { + if (dep?.sector && !declared.has(dep.sector)) { + errors.push( + `/dependencies/${i}/sector "${dep.sector}" is not a sector this pathway` + + ` declares (allowed: ${quote(declared)})`, + ); + } + }); + + return errors; +} diff --git a/testdata/valid/pathwayMetadata_v2_full.json b/testdata/valid/pathwayMetadata_v2_full.json new file mode 100644 index 00000000..fbae549f --- /dev/null +++ b/testdata/valid/pathwayMetadata_v2_full.json @@ -0,0 +1,254 @@ +{ + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", + "id": "pathway-v2-full", + "name": { + "full": "Full V2 Pathway", + "short": "Full V2" + }, + "publication": { + "title": { + "full": "Example Title", + "short": "Example" + }, + "subtitle": "Exercising every v2 shape", + "author": ["Doe, Jane"], + "publisher": { + "full": "TransitionZero" + }, + "year": 2024, + "month": 6, + "day": 1, + "city": "London", + "license": "CC BY 4.0", + "links": [ + { + "description": "Report", + "url": "https://www.example.com/" + } + ] + }, + "description": "A v2 pathway exercising multi-scope key features.", + "pathwayType": "Normative", + "modelTempIncrease": 1.5, + "modelYearStart": 2020, + "modelYearEnd": 2050, + "modelYearNetzero": 2050, + "geography": { + "global": true, + "regions": { + "South East Asia": ["ID", "TH", "VN"], + "North America": ["CA", "MX", "US"] + }, + "country": ["SG"] + }, + "sectors": [ + { + "name": "Power", + "technologies": ["Solar", "Wind", "Coal"], + "segments": [ + "Power generation", + "Energy storage", + "Transmission and distribution" + ] + }, + { + "name": "Steel", + "technologies": [], + "segments": ["Ironmaking", "Steelmaking"] + } + ], + "pathwayDescription": "A pathway whose key features vary by sector and geography, used to exercise the scoped-entry shape end to end.", + "metric": [ + "Emissions Intensity", + "Capacity", + "Generation", + "Technology Mix", + "Absolute Emissions" + ], + "keyFeatures": { + "emissionsTrajectory": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate decrease" + }, + { + "sector": "Power", + "geography": "South East Asia", + "value": "Significant decrease" + }, + { + "sector": "Steel", + "geography": "TH", + "value": "Minor decrease" + } + ], + "energyEfficiency": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Moderate improvement" + }, + { + "sector": "Steel", + "geography": "North America", + "value": "No information" + } + ], + "energyDemand": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Minor increase" + } + ], + "electrification": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Significant increase" + }, + { + "sector": "Power", + "geography": "SG", + "value": "Moderate increase" + } + ], + "policyTypes": [ + { + "sector": "across sectors", + "geography": "Global", + "value": ["Carbon price", "Subsidies"] + }, + { + "sector": "Power", + "geography": "South East Asia", + "value": ["Target technology shares", "Phaseout dates", "Other"] + } + ], + "technologyCostTrend": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Decrease" + } + ], + "emissionsScope": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "CO2e (Kyoto)" + } + ], + "policyAmbition": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "High ambition policies" + }, + { + "sector": "Steel", + "geography": "North America", + "value": "Current/legislated policies" + } + ], + "technologyCostsDetail": [ + { + "sector": "across sectors", + "geography": "Global", + "value": "Capital costs, O&M, etc." + } + ], + "newTechnologiesIncluded": [ + { + "sector": "across sectors", + "geography": "Global", + "value": ["CCUS", "DAC", "Green H2/ammonia", "Battery storage"] + }, + { + "sector": "Steel", + "geography": "Global", + "value": ["No new technologies"] + } + ], + "investmentNeeds": [] + }, + "coreDrivers": { + "policies": "Carbon pricing across both covered sectors.", + "emissionsTargets": "Net zero by 2050 with interim 2030 milestones.", + "technologyCosts": null, + "investmentChange": "Investment roughly doubles by 2040.", + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [ + { + "dependency_name": "Infrastructure and logistics", + "dependency_description": "Grid buildout must keep pace with renewable additions.", + "sector": "Power", + "evidence_type": "Quantitative" + }, + { + "dependency_name": "Technology", + "dependency_description": "Hydrogen direct reduction must reach commercial scale.", + "sector": "Steel", + "evidence_type": "Qualitative" + } + ], + "dataAvailability": { + "overall": "The full timeseries file covers all five metrics for Power at annual resolution from 2022 to 2050. Steel is described in the publication only.", + "byMetric": [ + { + "metricName": "Capacity", + "sector": "Power", + "sectorSegment": ["Fuel extraction and processing", "Power generation"], + "geography": ["Global", "South East Asia", "SG"], + "timeResolution": "1-year steps", + "dataFormat": "Tabular", + "granularity": ["Solar", "Wind", "Coal"], + "scopeLimitations": "Excludes off-grid generation. Reports no storage, so no storage breakdown applies." + }, + { + "metricName": "Generation", + "sector": "Power", + "sectorSegment": ["Energy storage"], + "geography": ["South East Asia"], + "timeResolution": "5-year steps", + "dataFormat": "Text", + "granularity": ["Unspecified"], + "scopeLimitations": "Battery storage only; pumped hydro is out of scope." + }, + { + "metricName": "Transmission lines", + "sector": "Power", + "sectorSegment": ["Transmission and distribution"], + "geography": ["SG"], + "timeResolution": "2050 data point", + "dataFormat": "Figure", + "granularity": ["International and national transmission lines"], + "scopeLimitations": "Grid losses only." + }, + { + "metricName": "Absolute emissions", + "sector": "Steel", + "sectorSegment": ["Ironmaking", "Steelmaking"], + "geography": ["Global"], + "timeResolution": "10-year steps", + "dataFormat": "Text", + "granularity": ["Scope 1 & 2"], + "scopeLimitations": "Ironmaking and steelmaking are modelled together and cannot be separated." + }, + { + "metricName": "Emissions intensity (total)", + "sector": "Steel", + "sectorSegment": ["Not covered"], + "geography": ["Not covered"], + "timeResolution": "Not covered", + "dataFormat": "Not covered", + "granularity": ["Not covered"], + "scopeLimitations": "Not covered" + } + ] + } +} diff --git a/testdata/valid/pathwayMetadata_v2_minimal.json b/testdata/valid/pathwayMetadata_v2_minimal.json new file mode 100644 index 00000000..6ea9eb22 --- /dev/null +++ b/testdata/valid/pathwayMetadata_v2_minimal.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", + "id": "pathway-v2-minimal", + "name": { "full": "Minimal V2 Pathway" }, + "publication": { + "title": { "full": "Example Title" }, + "publisher": { "full": "TransitionZero" }, + "year": 2024 + }, + "description": "A minimal v2 pathway file that passes schema validation.", + "pathwayType": "Exploratory", + "geography": { "regions": { "South East Asia": [] } }, + "sectors": [{ "name": "Other", "technologies": [] }], + "pathwayDescription": null, + "metric": ["Capacity"], + "keyFeatures": { + "emissionsTrajectory": [], + "energyEfficiency": [], + "energyDemand": [], + "electrification": [], + "policyTypes": [], + "technologyCostTrend": [], + "emissionsScope": [], + "policyAmbition": [], + "technologyCostsDetail": [], + "newTechnologiesIncluded": [], + "investmentNeeds": [] + }, + "coreDrivers": { + "policies": null, + "emissionsTargets": null, + "technologyCosts": null, + "investmentChange": null, + "macroeconomicDrivers": null, + "behavioralShifts": null, + "otherDrivers": null + }, + "dependencies": [] +}