diff --git a/REUSE.toml b/REUSE.toml index 2d928bc9df..7223df8518 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -9,6 +9,21 @@ precedence = "closest" SPDX-FileCopyrightText = "2026 Lusoris" SPDX-License-Identifier = "EUPL-1.2" +# The run-result timeline schema, its fixtures, its test and its docs page are +# contributor-authored (PR #2452): the fork-local EUPL-1.2 default applies, +# credited to their author. +[[annotations]] +path = [ + "api/run-result-timeline.schema.json", + "api/testdata/run-result-timeline/*.json", + "changelog.d/added/run-result-timeline-schema.md", + "docs/api/run-result-timeline.md", + "scripts/ci/tests/test_run_result_timeline_schema.py", +] +precedence = "closest" +SPDX-FileCopyrightText = "2026 luoye-cpu" +SPDX-License-Identifier = "EUPL-1.2" + # These six files descend from Netflix/vmaf root files. ADR-1250 requires # inherited work, and every fork contribution to it, to remain on the terms it # already carried rather than being swept into the fork-local EUPL default. diff --git a/api/run-result-timeline.schema.json b/api/run-result-timeline.schema.json new file mode 100644 index 0000000000..7abea8de21 --- /dev/null +++ b/api/run-result-timeline.schema.json @@ -0,0 +1,165 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/VMAFx/vmafx/api/run-result-timeline.schema.json", + "title": "VMAFx run-result timeline, schema version 1", + "description": "One versioned report format for a scoring run's provenance, per-frame scores, per-window statistics and events (VMAFx/vmafx#2452, RC5). It evolves additively under HISS-14: after a release, members are added, never removed, renamed or retyped; schema_version is the integer the document was written against, and a reader refuses a version it does not know by name. docs/api/run-result-timeline.md is the human reference. The skeleton pins the four sections' identities, not yet their full field contracts: provenance is the record of vmafx_context_provenance_json() (RC4 WP5, #2288; the authoritative field contract is the OpenAPI Provenance schema in api/openapi/vmafx-server-v1.yaml and the VmafxProvenance definition in core/api/vmafx.toml), and scoring_configuration is the option surface of #2258 (the scoring component group of the same OpenAPI document). The C ABI writer/reader and the byte-equal write-read-write identity check land in the follow-up PRs of the same issue.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "provenance", + "frames", + "windows", + "events" + ], + "properties": { + "schema_version": { + "const": 1, + "description": "The schema version the document was written against. 1 is the shape this file defines; a reader refuses an unknown version by name (#2452 acceptance)." + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "scoring_configuration": { + "$ref": "#/$defs/scoring_configuration" + }, + "frames": { + "type": "array", + "description": "The per-frame series, in ascending time-key order: one entry per scored frame.", + "items": { + "$ref": "#/$defs/frame_score" + } + }, + "windows": { + "type": "array", + "description": "Per-window statistics over ranges of the timeline: pooling and aggregation results.", + "items": { + "$ref": "#/$defs/window_statistic" + } + }, + "events": { + "type": "array", + "description": "The run's events, in ascending time-key order: alignment decisions, drops, repeats, freezes, switches and masked areas.", + "items": { + "$ref": "#/$defs/event" + } + } + }, + "$defs": { + "time_key": { + "description": "Where an entry sits on the timeline: at least one of the 0-based frame index, the presentation timestamp in nanoseconds from the run's zero, or the SMPTE-style timecode. A writer carries what the run knows - a frame score carries the frame index, a wall-clock event carries the timestamp - and an empty time key is refused.", + "type": "object", + "additionalProperties": false, + "anyOf": [ + { "required": ["frame_index"] }, + { "required": ["timestamp_ns"] }, + { "required": ["timecode"] } + ], + "properties": { + "frame_index": { + "type": "integer", + "minimum": 0, + "description": "0-based display order, the stream-metadata display_index of api/stream-metadata-fields.json." + }, + "timestamp_ns": { + "type": "integer", + "minimum": 0, + "description": "Presentation timestamp in nanoseconds from the run's zero." + }, + "timecode": { + "type": "string", + "pattern": "^[0-9]{2}:[0-9]{2}:[0-9]{2}(:[0-9]{2})?$", + "description": "SMPTE-style HH:MM:SS or HH:MM:SS:FF timecode." + } + } + }, + "provenance": { + "description": "The provenance record of the context that produced the run: vmafx_context_provenance_json() (RC4 WP5, #2288) carries one key per field of VmafxProvenance and its models, features and annotations. The authoritative field contract is the OpenAPI Provenance schema (api/openapi/vmafx-server-v1.yaml) and the VmafxProvenance definition in core/api/vmafx.toml; this schema pins the identity anchor so a timeline stays attributable and leaves the field contract to that single source.", + "type": "object", + "required": ["digest"], + "properties": { + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$", + "description": "The record's own digest: the digest member of VmafxProvenance, the SHA-256 of the record's canonical JSON (RFC 8785) without digest and elapsed_ns." + } + } + }, + "scoring_configuration": { + "description": "The scoring-option surface in effect for the run (#2258, WP8): the scoring component group of api/openapi/vmafx-server-v1.yaml. The skeleton leaves the members to that single source.", + "type": "object" + }, + "frame_score": { + "description": "One scored frame: its time key and the scores written for it.", + "type": "object", + "additionalProperties": false, + "required": ["time"], + "properties": { + "time": { + "$ref": "#/$defs/time_key" + }, + "scores": { + "type": "object", + "description": "Keys are feature names as the mounted models name them (for example vmaf, integer_adm2); values are that frame's numbers for the feature. A frame without scores is an empty object.", + "additionalProperties": { + "type": "number" + } + } + } + }, + "window_statistic": { + "description": "One statistic computed over a window of the timeline; the range is the half-open interval the writer knows (start inclusive, end exclusive when both ends are frame indices).", + "type": "object", + "additionalProperties": false, + "required": ["start", "end", "statistic", "value"], + "properties": { + "start": { + "$ref": "#/$defs/time_key" + }, + "end": { + "$ref": "#/$defs/time_key" + }, + "statistic": { + "type": "string", + "minLength": 1, + "description": "The statistic's name as the writer computes it (for example pooled, mean, worst_frame)." + }, + "feature": { + "type": "string", + "minLength": 1, + "description": "The feature the statistic belongs to; omitted for a run-level statistic." + }, + "value": { + "type": ["number", "string"], + "description": "The computed value as a JSON number; a writer spells a non-finite result as the string NaN, +Inf or -Inf, and a reader matches that spelling." + } + } + }, + "event": { + "description": "One run event. A planted event without a time key is refused by name (#2452 acceptance).", + "type": "object", + "additionalProperties": false, + "required": ["type", "time"], + "properties": { + "type": { + "enum": [ + "alignment_decision", + "drop", + "repeat", + "freeze", + "switch", + "masked_area" + ], + "description": "alignment_decision records how streams were aligned; drop, repeat and freeze record frame-handling decisions; switch records a source or configuration switch mid-run; masked_area records a region excluded from scoring. The vocabulary grows additively under HISS-14." + }, + "time": { + "$ref": "#/$defs/time_key" + }, + "detail": { + "type": "object", + "description": "The event's facts as the writer reports them (for example the alignment offsets of an alignment_decision); the skeleton constrains nothing further." + } + } + } + } +} diff --git a/api/testdata/run-result-timeline/invalid-event-without-time.json b/api/testdata/run-result-timeline/invalid-event-without-time.json new file mode 100644 index 0000000000..2f66e1f070 --- /dev/null +++ b/api/testdata/run-result-timeline/invalid-event-without-time.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "provenance": { + "digest": "sha256:1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809" + }, + "frames": [ + { + "time": { "frame_index": 0 }, + "scores": { "vif": 0.95102 } + } + ], + "windows": [], + "events": [ + { + "type": "drop", + "detail": { "reason": "duplicate pts" } + } + ] +} diff --git a/api/testdata/run-result-timeline/invalid-unknown-schema-version.json b/api/testdata/run-result-timeline/invalid-unknown-schema-version.json new file mode 100644 index 0000000000..cb23f10c8b --- /dev/null +++ b/api/testdata/run-result-timeline/invalid-unknown-schema-version.json @@ -0,0 +1,9 @@ +{ + "schema_version": 2, + "provenance": { + "digest": "sha256:1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809" + }, + "frames": [], + "windows": [], + "events": [] +} diff --git a/api/testdata/run-result-timeline/valid-full.json b/api/testdata/run-result-timeline/valid-full.json new file mode 100644 index 0000000000..9fdb4e0ff1 --- /dev/null +++ b/api/testdata/run-result-timeline/valid-full.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "provenance": { + "abi_major": 0, + "abi_minor": 2, + "abi_patch": 0, + "active_backend": 1, + "version": "v0.2.0-rc.1-3-g5c941077b", + "commit": "5c941077bd0f8f68d2c1430e548c902a5245a8ab8", + "build_id": "sha256:0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + "compiler": "c=gcc 15.2.1 cpp=gcc 15.2.1", + "backends": "cpu,cuda", + "simd": "avx2", + "device_index": 0, + "device_name": "cpu", + "device_runtime": "x86_64", + "n_frames": 2, + "digest": "sha256:1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809", + "elapsed_ns": 240000000 + }, + "scoring_configuration": { + "model": "vmaf_v1.0.16", + "backend": "cpu", + "threads": 8 + }, + "frames": [ + { + "time": { "frame_index": 0, "timestamp_ns": 0, "timecode": "00:00:00:00" }, + "scores": { "integer_adm2": 0.98213, "vif": 0.95102, "motion": 8.41 } + }, + { + "time": { "frame_index": 1, "timestamp_ns": 41708333, "timecode": "00:00:00:01" }, + "scores": { "integer_adm2": 0.97747, "vif": 0.94855, "motion": 9.12 } + } + ], + "windows": [ + { + "start": { "frame_index": 0 }, + "end": { "frame_index": 2 }, + "statistic": "pooled", + "feature": "integer_adm2", + "value": 0.9798 + }, + { + "start": { "frame_index": 0 }, + "end": { "frame_index": 2 }, + "statistic": "worst_frame", + "feature": "vif", + "value": 0.94855 + } + ], + "events": [ + { + "type": "alignment_decision", + "time": { "timecode": "00:00:00:00" }, + "detail": { "offset_frames": 0, "method": "first-frame anchor" } + }, + { + "type": "drop", + "time": { "frame_index": 1, "timestamp_ns": 41708333 }, + "detail": { "reason": "duplicate pts" } + }, + { + "type": "masked_area", + "time": { "frame_index": 0 }, + "detail": { "region": "bottom-bar", "reason": "burned-in caption" } + } + ] +} diff --git a/api/testdata/run-result-timeline/valid-minimal.json b/api/testdata/run-result-timeline/valid-minimal.json new file mode 100644 index 0000000000..1d6ad5e6fa --- /dev/null +++ b/api/testdata/run-result-timeline/valid-minimal.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "provenance": { + "digest": "sha256:1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809" + }, + "frames": [], + "windows": [], + "events": [] +} diff --git a/changelog.d/added/run-result-timeline-schema.md b/changelog.d/added/run-result-timeline-schema.md new file mode 100644 index 0000000000..079318ddd6 --- /dev/null +++ b/changelog.d/added/run-result-timeline-schema.md @@ -0,0 +1 @@ +- **The RC5 run-result timeline has a schema skeleton.** `api/run-result-timeline.schema.json` defines the versioned document that will carry one run's provenance, per-frame scores, per-window statistics and events across the CLI, the C API and the server (#2452); it evolves additively under HISS-14, and the acceptance fixtures in `api/testdata/run-result-timeline/` pin the shape in `scripts/ci/tests/test_run_result_timeline_schema.py`. The writer/reader C ABI and the byte-equal write-read-write identity check land in the follow-up PRs of the same issue. diff --git a/docs/api/run-result-timeline.md b/docs/api/run-result-timeline.md new file mode 100644 index 0000000000..51c61dda48 --- /dev/null +++ b/docs/api/run-result-timeline.md @@ -0,0 +1,60 @@ + +# Run-result timeline — `api/run-result-timeline.schema.json` + +VMAFx plans to write one report format for a scoring run instead of letting +alignment, fields, masks and bits-per-pixel each invent their own +([VMAFx/vmafx#2452](https://github.com/VMAFx/vmafx/issues/2452), RC5): a +versioned JSON document that carries the run's provenance, its per-frame +scores, its per-window statistics and its events - alignment decisions, drops, +repeats, freezes, switches, masked areas - each located by one time key that +can carry a frame index, a timestamp or a timecode. + +[`api/run-result-timeline.schema.json`](https://github.com/VMAFx/vmafx/blob/master/api/run-result-timeline.schema.json) +is the contract of that document, and this page is its human reference. It is +the shape only, not the reporting code: no CLI flag, C call or server endpoint +writes the document yet. The writer/reader C ABI and the identity checks land +in the follow-up PRs of the same issue; the converters from the existing +XML/JSON/CSV outputs follow them. + +## What the document holds + +| Section | Carries | Contract | +| --- | --- | --- | +| `schema_version` | The schema version the document was written against (`1` today) | A reader refuses an unknown version by name | +| `provenance` | The provenance record of the context that produced the scores (`vmafx_context_provenance_json()`, RC4 WP5, #2288) | The authoritative field contract is the OpenAPI [`Provenance` schema](../../api/openapi/vmafx-server-v1.yaml) and the `VmafxProvenance` definition in [`core/api/vmafx.toml`](../../core/api/vmafx.toml); the timeline schema pins the `digest` identity anchor | +| `scoring_configuration` | The scoring-option surface in effect (#2258, WP8) | The `scoring` component group of the same OpenAPI document is the single source | +| `frames` | The per-frame series: a time key plus the frame's per-feature scores | The time key carries at least one of frame index, timestamp, timecode | +| `windows` | Per-window statistics: a start, an end, the statistic's name and its value | Ranges are the half-open interval the writer knows | +| `events` | The run's events: a type from the additive vocabulary, a required time key, optional facts | An event without a time key is refused by name | + +The time key mirrors what a run actually knows: a frame score carries the +0-based display-order frame index (the stream-metadata `display_index` of +[`api/stream-metadata-fields.json`](https://github.com/VMAFx/vmafx/blob/master/api/stream-metadata-fields.json)), +a wall-clock event carries the nanosecond timestamp, and a writer may add the +SMPTE-style timecode when the run has one. + +## Versioning + +The document evolves additively under the project's +[Append-Only ABI](../principles.md) invariant (HISS-14): after a release, the +schema gains members, and never removes, renames or retypes one. `schema_version` +is the integer a document was written against; a reader that meets a version it +does not know refuses the document by name rather than guessing. The acceptance +fixtures that pin today's shape live in +[`api/testdata/run-result-timeline/`](https://github.com/VMAFx/vmafx/tree/master/api/testdata/run-result-timeline) +and run in +[`scripts/ci/tests/test_run_result_timeline_schema.py`](https://github.com/VMAFx/vmafx/blob/master/scripts/ci/tests/test_run_result_timeline_schema.py) +(the `tooling` suite): + +```bash +python -m pytest scripts/ci/tests/test_run_result_timeline_schema.py -q +``` + +## Not in the skeleton + +The skeleton pins identities, not every field contract. The provenance record's +fields and the scoring-option surface stay with their single sources (the +OpenAPI document and `core/api/vmafx.toml`) instead of being copied here, so +the two contracts cannot drift; the timeline schema constrains what makes a +timeline attributable and addressable. The richer verdict and facts machinery +of #2311 extends this schema in 1.1. diff --git a/docs/rebase-notes.d/run-result-timeline-schema.md b/docs/rebase-notes.d/run-result-timeline-schema.md new file mode 100644 index 0000000000..22010280eb --- /dev/null +++ b/docs/rebase-notes.d/run-result-timeline-schema.md @@ -0,0 +1,17 @@ +## Run-result timeline schema skeleton (2026-11-10) + +New files only; nothing upstream corresponds to them, so a rebase cannot +collide on their content. Two rebase-sensitive surfaces: + +- `REUSE.toml` — the contributor annotation block sits directly after the + first `[[annotations]]` catch-all. Other contributor pull requests (#3317 + and later) append their blocks at the same position; a conflict resolves by + keeping both blocks, catch-all first and contributor blocks after it. Never + fold a contributor block into the catch-all: the catch-all credits + `2026 Lusoris`, and folding would misattribute contributor files to the + maintainer (the credits gate reads the file headers against `REUSE.toml`). +- `api/run-result-timeline.schema.json` — the document evolves additively + under HISS-14. A rebase that removes, renames or retypes a schema member, + or reorders the `schema_version` contract, is a break of the additive + contract, not a conflict to clean up; restore the schema side and reopen + the discussion instead. diff --git a/scripts/ci/tests/test_run_result_timeline_schema.py b/scripts/ci/tests/test_run_result_timeline_schema.py new file mode 100644 index 0000000000..890706d307 --- /dev/null +++ b/scripts/ci/tests/test_run_result_timeline_schema.py @@ -0,0 +1,86 @@ +# Copyright 2026 luoye-cpu +# SPDX-License-Identifier: EUPL-1.2 +"""Acceptance fixtures for api/run-result-timeline.schema.json (issue #2452). + +The schema is the skeleton the RC5 run-result timeline grows from; these +fixtures pin the shape the issue's acceptance names: + +- the boundary document - every section present, all series empty - validates; +- the positive document, which carries entries in every section, validates; +- a planted event without a time key is refused by name (the refusal points at + the offending event); +- an unknown schema version is refused by name (the refusal points at + schema_version). + +The byte-equal write-read-write round trip needs the writer/reader C ABI and +lands with it in a later PR of the same issue. +""" + +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from typing import Any + +try: + import jsonschema + + _HAS_JSONSCHEMA = True +except ImportError: # the tooling-tests lock installs it; stand-alone runs may not + _HAS_JSONSCHEMA = False + +ROOT = Path(__file__).resolve().parents[3] +SCHEMA_PATH = ROOT / "api" / "run-result-timeline.schema.json" +FIXTURE_DIR = ROOT / "api" / "testdata" / "run-result-timeline" + + +@unittest.skipUnless(_HAS_JSONSCHEMA, "jsonschema is not installed") +class RunResultTimelineSchemaTests(unittest.TestCase): + """The shape rules the #2452 skeleton must hold.""" + + @classmethod + def setUpClass(cls) -> None: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + cls.validator: Any = jsonschema.Draft202012Validator(schema) + + def load(self, name: str) -> dict: + return json.loads((FIXTURE_DIR / name).read_text(encoding="utf-8")) + + def test_valid_minimal_document(self) -> None: + """Boundary: every section present, all series empty.""" + self.validator.validate(self.load("valid-minimal.json")) + + def test_valid_full_document(self) -> None: + """Positive: provenance, scoring configuration, frames, windows, events.""" + self.validator.validate(self.load("valid-full.json")) + + def test_event_without_time_key_refused_by_name(self) -> None: + """A planted event without a time key is refused, naming the event.""" + errors = list(self.validator.iter_errors(self.load("invalid-event-without-time.json"))) + self.assertTrue(errors, "the event without a time key must be refused") + named = [ + error + for error in errors + if error.json_path.startswith("$.events") and "time" in error.message + ] + self.assertTrue( + named, + "the refusal must name the offending event and its missing time key; " + f"got: {[error.json_path for error in errors]}", + ) + + def test_unknown_schema_version_refused_by_name(self) -> None: + """An unknown schema version is refused, naming the member.""" + errors = list(self.validator.iter_errors(self.load("invalid-unknown-schema-version.json"))) + self.assertTrue(errors, "the unknown schema version must be refused") + named = [error for error in errors if "schema_version" in error.json_path] + self.assertTrue( + named, + "the refusal must name schema_version; " + f"got: {[error.json_path for error in errors]}", + ) + + +if __name__ == "__main__": + unittest.main()