From a50308daf6fdf7b2a7fddb1ff08c94dbd1d9ae9c Mon Sep 17 00:00:00 2001 From: malchul Date: Sun, 7 Jun 2026 22:14:14 +0300 Subject: [PATCH 1/6] Added hparams storing --- apps/web/content/docs/domains/experiments.md | 12 +- .../content/docs/sdk/experiment-logging.md | 34 +- apps/web/package.json | 2 +- .../projects/[projectId]/compare/page.tsx | 2 +- .../components/shared/experiment-diff-ui.tsx | 61 +++ .../shared/experiment-features-panel.tsx | 79 +--- .../shared/experiment-hparams-panel.tsx | 248 ++++++++++++ .../shared/experiment-hparams-tree.tsx | 232 +++++++++++ .../components/shared/experiment-sidebar.tsx | 22 +- .../compare/components/compare-shell.tsx | 120 +++++- .../experiment-data-compare-table.test.ts | 25 ++ .../experiment-data-compare-table.tsx | 362 ++++++++++++++++++ .../components/experiment-name-tooltip.tsx | 26 ++ .../src/domain/compare/components/index.ts | 1 + apps/web/src/domain/compare/hooks/index.ts | 1 + .../use-experiment-data-compare-layout.ts | 114 ++++++ .../components/hparams-compare-tab.tsx | 126 ++++++ .../compare/hparams/components/index.ts | 1 + .../hparams/lib/hparams-compare.test.ts | 21 + .../compare/hparams/lib/hparams-compare.ts | 58 +++ .../services/hparams-compare-service.ts | 16 + .../domain/compare/hparams/services/index.ts | 1 + .../compare/hparams/types/hparams-compare.ts | 12 + .../src/domain/compare/hparams/types/index.ts | 1 + .../components/code-viewer.tsx | 0 .../components/collapsible-sidebar.tsx | 0 .../components/diff-viewer.tsx | 0 .../components/expand-unchanged-control.tsx | 2 +- .../components/file-compare-view.tsx | 22 +- .../components/file-comparison.tsx | 0 .../{ => snapshots}/components/file-tree.tsx | 0 .../components/file-viewer.tsx | 0 .../components/files-compare-tab.tsx | 39 +- .../compare/snapshots/components/index.ts | 1 + .../components/inline-diff-text.tsx | 0 .../components/syntax-highlighted-code.tsx | 0 .../compare/{ => snapshots}/lib/diff.ts | 0 .../compare/{ => snapshots}/lib/file-tree.ts | 0 .../compare/snapshots/services/index.ts | 1 + .../services/snapshot-compare-service.ts} | 31 +- .../domain/compare/snapshots/types/index.ts | 1 + .../types/snapshot-compare.ts} | 0 .../hooks/experiment-hparams-hook.ts | 50 +++ .../web/src/domain/experiments/hooks/index.ts | 1 + .../experiments/lib/hparams-diff.test.ts | 21 + .../domain/experiments/lib/hparams-diff.ts | 48 +++ .../experiments/lib/hparams-json.test.ts | 12 + .../domain/experiments/lib/hparams-json.ts | 20 + .../services/experiment-hparams-service.ts | 28 ++ .../services/experiment-snapshots-service.ts | 23 ++ .../src/domain/experiments/services/index.ts | 1 + .../src/domain/experiments/types/hparams.ts | 18 + .../web/src/domain/experiments/types/index.ts | 3 +- apps/web/src/lib/constants/api-routes.ts | 4 + apps/web/src/lib/constants/query-keys.ts | 3 + .../src/{domain/compare => lib}/downloads.ts | 0 examples/pytorch-mnist-tensorboardx/uv.lock | 18 +- examples/training/train.py | 58 +++ ...07_001_add_hparams_experiment_data_type.py | 26 ++ python/backend/pyproject.toml | 2 +- .../src/api/routes/service_dependencies.py | 2 + .../src/domain/experiment_data/controller.py | 43 ++- .../backend/src/domain/experiment_data/dto.py | 47 ++- .../src/domain/experiment_data/mapper.py | 15 +- .../src/domain/experiment_data/service.py | 111 +++++- .../backend/src/domain/projects/controller.py | 28 +- python/backend/src/models.py | 1 + .../experiment_data/test_hparams_service.py | 155 ++++++++ python/backend/uv.lock | 4 +- python/object_storage/pyproject.toml | 2 +- python/object_storage/uv.lock | 4 +- python/scalars_service/pyproject.toml | 2 +- python/scalars_service/uv.lock | 4 +- python/sdk/pyproject.toml | 2 +- .../src/experiment_tracker_sdk/__init__.py | 4 +- .../client/domain/experiment_data/dto.py | 13 + .../client/domain/experiment_data/service.py | 16 + .../sdk/src/experiment_tracker_sdk/error.py | 6 + .../src/experiment_tracker_sdk/exp_tracker.py | 12 + .../sdk/src/experiment_tracker_sdk/hparams.py | 86 +++++ .../utils/hooks/tensorboard.py | 61 +-- .../sdk/tests/test_client_spec_and_prefix.py | 16 +- python/sdk/tests/test_exp_tracker_metrics.py | 22 ++ python/sdk/tests/test_hparams.py | 65 ++++ python/sdk/tests/test_tensorboard_hook.py | 19 +- python/sdk/tests/test_version.py | 12 + python/shared/pyproject.toml | 2 +- 87 files changed, 2516 insertions(+), 248 deletions(-) create mode 100644 apps/web/src/components/shared/experiment-diff-ui.tsx create mode 100644 apps/web/src/components/shared/experiment-hparams-panel.tsx create mode 100644 apps/web/src/components/shared/experiment-hparams-tree.tsx create mode 100644 apps/web/src/domain/compare/components/experiment-data-compare-table.test.ts create mode 100644 apps/web/src/domain/compare/components/experiment-data-compare-table.tsx create mode 100644 apps/web/src/domain/compare/components/experiment-name-tooltip.tsx create mode 100644 apps/web/src/domain/compare/components/index.ts create mode 100644 apps/web/src/domain/compare/hooks/index.ts create mode 100644 apps/web/src/domain/compare/hooks/use-experiment-data-compare-layout.ts create mode 100644 apps/web/src/domain/compare/hparams/components/hparams-compare-tab.tsx create mode 100644 apps/web/src/domain/compare/hparams/components/index.ts create mode 100644 apps/web/src/domain/compare/hparams/lib/hparams-compare.test.ts create mode 100644 apps/web/src/domain/compare/hparams/lib/hparams-compare.ts create mode 100644 apps/web/src/domain/compare/hparams/services/hparams-compare-service.ts create mode 100644 apps/web/src/domain/compare/hparams/services/index.ts create mode 100644 apps/web/src/domain/compare/hparams/types/hparams-compare.ts create mode 100644 apps/web/src/domain/compare/hparams/types/index.ts rename apps/web/src/domain/compare/{ => snapshots}/components/code-viewer.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/collapsible-sidebar.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/diff-viewer.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/expand-unchanged-control.tsx (89%) rename apps/web/src/domain/compare/{ => snapshots}/components/file-compare-view.tsx (98%) rename apps/web/src/domain/compare/{ => snapshots}/components/file-comparison.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/file-tree.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/file-viewer.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/files-compare-tab.tsx (94%) create mode 100644 apps/web/src/domain/compare/snapshots/components/index.ts rename apps/web/src/domain/compare/{ => snapshots}/components/inline-diff-text.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/components/syntax-highlighted-code.tsx (100%) rename apps/web/src/domain/compare/{ => snapshots}/lib/diff.ts (100%) rename apps/web/src/domain/compare/{ => snapshots}/lib/file-tree.ts (100%) create mode 100644 apps/web/src/domain/compare/snapshots/services/index.ts rename apps/web/src/domain/compare/{service.ts => snapshots/services/snapshot-compare-service.ts} (64%) create mode 100644 apps/web/src/domain/compare/snapshots/types/index.ts rename apps/web/src/domain/compare/{types.ts => snapshots/types/snapshot-compare.ts} (100%) create mode 100644 apps/web/src/domain/experiments/hooks/experiment-hparams-hook.ts create mode 100644 apps/web/src/domain/experiments/lib/hparams-diff.test.ts create mode 100644 apps/web/src/domain/experiments/lib/hparams-diff.ts create mode 100644 apps/web/src/domain/experiments/lib/hparams-json.test.ts create mode 100644 apps/web/src/domain/experiments/lib/hparams-json.ts create mode 100644 apps/web/src/domain/experiments/services/experiment-hparams-service.ts create mode 100644 apps/web/src/domain/experiments/types/hparams.ts rename apps/web/src/{domain/compare => lib}/downloads.ts (100%) create mode 100644 python/backend/alembic/versions/20260607_001_add_hparams_experiment_data_type.py create mode 100644 python/backend/tests/domain/experiment_data/test_hparams_service.py create mode 100644 python/sdk/src/experiment_tracker_sdk/hparams.py create mode 100644 python/sdk/tests/test_hparams.py create mode 100644 python/sdk/tests/test_version.py diff --git a/apps/web/content/docs/domains/experiments.md b/apps/web/content/docs/domains/experiments.md index 1ec541d6..70d9bce6 100644 --- a/apps/web/content/docs/domains/experiments.md +++ b/apps/web/content/docs/domains/experiments.md @@ -46,9 +46,9 @@ tracker.progress(100) Experiment colors make runs visually distinct. They are especially important on scalar plots, where multiple experiments may share the same chart. Colors must be hex strings such as `#3366cc` or `#3366ccff`. -## Features and parent diffs +## Features, hyperparameters, and parent diffs -Features are a tree of named nodes. Today they are the closest equivalent to hyperparameters and run configuration in the product. +Features are a tree of named nodes that describe semantic experiment changes. ```python tracker.features([ @@ -66,7 +66,13 @@ tracker.features([ ]) ``` -When an experiment has a parent, the sidebar can show differences from the parent experiment and let users edit the feature tree. Use this for "what changed in this run?" information until a dedicated hyperparameter domain exists. +When an experiment has a parent, the sidebar can show differences from the parent +experiment and let users edit the feature tree. Use this for "what changed in this +run?" information. + +Hyperparameters are stored separately as nested JSON and can be logged through +`tracker.log_hparams(...)` or edited from the experiment sidebar. The Compare page +shows baseline-relative added, removed, and changed hyperparameter values. ## Parent experiments and DAG diff --git a/apps/web/content/docs/sdk/experiment-logging.md b/apps/web/content/docs/sdk/experiment-logging.md index b442e233..376bb610 100644 --- a/apps/web/content/docs/sdk/experiment-logging.md +++ b/apps/web/content/docs/sdk/experiment-logging.md @@ -184,10 +184,18 @@ tracker.features([ {"name": "data", "children": [{"name": "mnist"}]}, {"name": "model", "children": [{"name": "small-cnn"}]}, ]) +tracker.log_hparams({ + "optimizer": {"name": "adamw", "lr": 3e-4}, + "training": {"batch_size": 32, "epochs": 50}, +}) ``` `progress` accepts `0..100` integers or `0..1` floats. `parent_experiment` resolves by name or id inside the current project. +`features` describe semantic changes and research ideas. `log_hparams` stores the +configurable training values used by the run. Each `log_hparams` call fully replaces +the experiment's previous hyperparameter document. + ## Run the repository example ```bash @@ -572,12 +580,36 @@ tracker.features([ ]) ``` -Updates the experiment feature tree. Features are currently the best place to store hyperparameter-like run structure and "what changed" information. +Updates the experiment feature tree. Use features for semantic changes, ablations, +new mechanisms, and "what changed" information rather than configurable training +values. When an experiment has a parent, the sidebar can show feature differences from that parent. --- +### `log_hparams(...)` + +```python +tracker.log_hparams({ + "optimizer": { + "name": "adamw", + "lr": 0.001, + }, + "training": { + "batch_size": 64, + "seed": 42, + }, +}) +``` + +Validates and stores a nested hyperparameter JSON object. Repeated calls fully replace +the previous document; they do not deep-merge it. Common values such as `Path`, `Enum`, +`date`, `datetime`, and NumPy scalar values are converted when safe. Unsupported values +raise `HparamsSerializationError` with the failing parameter path. + +--- + ### `name(...)` ```python diff --git a/apps/web/package.json b/apps/web/package.json index 81a916e6..7686e5de 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "web", - "version": "0.11.8", + "version": "0.12.0", "private": true, "scripts": { "dev": "next dev", diff --git a/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx b/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx index b6918c39..1dc56a18 100644 --- a/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx +++ b/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useParams } from "next/navigation"; -import { CompareShell } from "@/domain/compare/components/compare-shell"; +import { CompareShell } from "@/domain/compare/components"; export default function ComparePage() { const { projectId } = useParams<{ projectId: string }>(); diff --git a/apps/web/src/components/shared/experiment-diff-ui.tsx b/apps/web/src/components/shared/experiment-diff-ui.tsx new file mode 100644 index 00000000..216c1c27 --- /dev/null +++ b/apps/web/src/components/shared/experiment-diff-ui.tsx @@ -0,0 +1,61 @@ +import type { ReactNode } from "react"; +import { CircleMinus, CirclePlus, PencilLine } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; + +export type ExperimentDiffStatus = "unchanged" | "added" | "removed" | "changed"; + +export function ExperimentDiffCountBadge({ + status, + label, + value, + compact = false, +}: { + status: Exclude; + label: string; + value: number; + compact?: boolean; +}) { + const marker = status === "added" ? "+" : status === "removed" ? "-" : "~"; + return ( + + {compact ? `${marker}${value}` : `${label} ${value}`} + + ); +} + +export function ExperimentDiffIcon({ + status, + title, +}: { + status: ExperimentDiffStatus; + title?: string; +}) { + const iconClassName = "h-3.5 w-3.5"; + let icon: ReactNode = null; + if (status === "added") { + icon = ; + } else if (status === "removed") { + icon = ; + } else if (status === "changed") { + icon = ; + } + return icon ? ( + + {icon} + + ) : null; +} + +export function experimentDiffSurfaceClass(status: ExperimentDiffStatus): string { + if (status === "added") return "bg-green-500/10 text-green-800 dark:text-green-300"; + if (status === "removed") return "bg-red-500/10 text-red-800 dark:text-red-300"; + if (status === "changed") return "bg-amber-500/10 text-amber-800 dark:text-amber-300"; + return "text-foreground/80"; +} + +function experimentDiffBadgeClass(status: Exclude): string { + if (status === "added") return "border-green-500/20 bg-green-500/10 text-green-700"; + if (status === "removed") return "border-red-500/20 bg-red-500/10 text-red-700"; + return "border-amber-500/20 bg-amber-500/10 text-amber-700"; +} diff --git a/apps/web/src/components/shared/experiment-features-panel.tsx b/apps/web/src/components/shared/experiment-features-panel.tsx index 9f4c2fc4..cd7927d1 100644 --- a/apps/web/src/components/shared/experiment-features-panel.tsx +++ b/apps/web/src/components/shared/experiment-features-panel.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { ArrowLeftRight, ArrowRight, CircleMinus, CirclePlus, GitCompare, Maximize2, PencilLine, Save } from "lucide-react"; +import { ArrowLeftRight, ArrowRight, GitCompare, Maximize2, PencilLine, Save } from "lucide-react"; import type { Experiment } from "@/domain/experiments/types"; import { experimentsService } from "@/domain/experiments/services"; import { QUERY_KEYS } from "@/lib/constants/query-keys"; @@ -15,7 +15,6 @@ import { type FeatureNode, } from "@/lib/features/feature-tree"; import { Button } from "@/components/ui/button"; -import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Dialog, @@ -34,6 +33,11 @@ import { } from "@/components/ui/select"; import { FeatureBulletEditor } from "@/components/shared/feature-bullet-editor"; import { FeatureEditorLabelWithHelp } from "@/components/shared/feature-editor-help"; +import { + ExperimentDiffCountBadge, + ExperimentDiffIcon, + experimentDiffSurfaceClass, +} from "@/components/shared/experiment-diff-ui"; type ExperimentFeaturesPanelProps = { experiment: Experiment; @@ -127,9 +131,9 @@ export function ExperimentFeaturesPanel({ {parentExperiment && showDiffs ? (
- - - + + +
) : null} - {label} {value} - - ); -} - function FeatureView({ childFeatures, parentExperimentExists, @@ -240,9 +228,9 @@ function FeatureExpandedModal({ {showDiffs ? (
- - - + + +
) : null} @@ -330,7 +318,7 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) { if ((row.status === "renamed" || row.status === "changed") && row.parentName && row.childName) { return ( } + icon={} previousName={row.parentName} name={row.childName} depth={row.depth} @@ -340,15 +328,10 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) { return ( } + icon={} name={displayName} depth={row.depth} - className={cn( - row.status === "added" && "bg-green-500/10 text-green-800 dark:text-green-300", - row.status === "removed" && "bg-red-500/10 text-red-800 dark:text-red-300", - (row.status === "renamed" || row.status === "changed") && - "bg-amber-500/10 text-amber-800 dark:text-amber-300" - )} + className={experimentDiffSurfaceClass(row.status === "renamed" ? "changed" : row.status)} /> ); } @@ -410,30 +393,6 @@ function FeatureNodeDot() { return