From a72054a074b2d9a8fd32669e66a2f3860931b5bf Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 10:39:32 -0500 Subject: [PATCH 01/11] feat: set Blender scene fps from footage frame rate Scene frames are sync indices, so a 60 fps capture played at Blender's 24 fps default runs 2.5x slow and the camera backgrounds drift against the skeleton. write_blender_scene now requires an fps argument and stamps it into scene.render.fps. --- scripts/demo/demo_markerless_calibration.py | 1 + src/caliscope/export/blender_scene.py | 9 +++++++++ tests/test_blender_export.py | 6 ++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/demo/demo_markerless_calibration.py b/scripts/demo/demo_markerless_calibration.py index 7eea464e1..b4e1a3532 100644 --- a/scripts/demo/demo_markerless_calibration.py +++ b/scripts/demo/demo_markerless_calibration.py @@ -191,6 +191,7 @@ volume.camera_array, world_points, OUTPUT_DIR / "capture_volume_scene.py", + fps=60, # Pose2Sim Demo_SinglePerson footage videos=videos, wireframe=tracker_registry.wireframe_for(TRACKER_KEY), ) diff --git a/src/caliscope/export/blender_scene.py b/src/caliscope/export/blender_scene.py index 8b78799f8..bccc8f331 100644 --- a/src/caliscope/export/blender_scene.py +++ b/src/caliscope/export/blender_scene.py @@ -185,6 +185,10 @@ def main(): frames = PAYLOAD["frames"] scene.frame_start = frames[0] scene.frame_end = frames[-1] + # Scene frames are sync indices at the footage frame rate. Without this, + # Blender plays at its 24 fps default and 60 fps footage runs 2.5x slow. + scene.render.fps = int(round(PAYLOAD["fps"])) + scene.render.fps_base = 1.0 first = PAYLOAD["cameras"][0] scene.render.resolution_x = int(first["width"]) scene.render.resolution_y = int(first["height"]) @@ -314,6 +318,7 @@ def write_blender_scene( world_points: WorldPoints, output_path: Path | str, *, + fps: float, videos: Mapping[int, Path | str] | None = None, wireframe: WireFrameView | None = None, run_blender: bool = True, @@ -326,6 +331,9 @@ def write_blender_scene( sync indices, so pass full-frame triangulations for smooth playback). output_path: Path for the generated scene script (.py). The .blend is saved beside it with the same stem. + fps: Footage frame rate. Sets the scene playback rate so scene frames + (sync indices) play in real time. Must match the source video, or + playback and the camera backgrounds run at the wrong speed. videos: Optional cam_id to video path mapping; matching cameras get their footage as a frame-synced camera background. wireframe: Optional skeleton topology; colors the keypoint dots by @@ -378,6 +386,7 @@ def write_blender_scene( "joints": np.round(positions, 5).tolist() if edges else [], "edges": edges, "cameras": cameras_payload, + "fps": float(fps), "keypoint_radius_m": KEYPOINT_RADIUS_M, "floor_size_m": FLOOR_SIZE_M, } diff --git a/tests/test_blender_export.py b/tests/test_blender_export.py index f526070e6..050e5e341 100644 --- a/tests/test_blender_export.py +++ b/tests/test_blender_export.py @@ -43,7 +43,7 @@ def test_writes_compilable_scene_script(tmp_path): point_names={"a": 0, "b": 1}, ) script_path = write_blender_scene( - cameras, _world_points(), tmp_path / "scene.py", wireframe=wireframe, run_blender=False + cameras, _world_points(), tmp_path / "scene.py", fps=60, wireframe=wireframe, run_blender=False ) source = script_path.read_text() @@ -52,6 +52,8 @@ def test_writes_compilable_scene_script(tmp_path): payload_json = source.split('json.loads(r"""')[1].split('""")')[0] payload = json.loads(payload_json) assert payload["frames"] == [0, 3, 6] + assert payload["fps"] == 60 + assert "scene.render.fps" in source # scene plays at footage rate, not Blender's 24 fps default assert len(payload["cameras"]) == 2 assert payload["edges"] == [[0, 1]] group_names = {group["name"] for group in payload["groups"]} @@ -61,4 +63,4 @@ def test_writes_compilable_scene_script(tmp_path): def test_rejects_unposed_cameras(tmp_path): cameras = CameraArray({1: CameraData.from_intrinsics(cam_id=1, size=(1280, 720), focal_length=900.0)}) with pytest.raises(ValueError, match="posed"): - write_blender_scene(cameras, _world_points(), tmp_path / "scene.py", run_blender=False) + write_blender_scene(cameras, _world_points(), tmp_path / "scene.py", fps=60, run_blender=False) From e2932eb99d1e46e3aa28f9c18f0b7f3233c7c36b Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 10:39:37 -0500 Subject: [PATCH 02/11] docs: add OpenCap markerless calibration demo script Runs the markerless pipeline on OpenCap LabValidation footage and scores the recovered poses against OpenCap's own checkerboard calibration. The syncdWithMocap videos carry a constant per-camera frame offset, so the script measures it by keypoint cross-correlation and settles the integer by reprojection error before calibrating. --- .gitignore | 1 + scripts/demo/demo_opencap_calibration.py | 456 +++++++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 scripts/demo/demo_opencap_calibration.py diff --git a/.gitignore b/.gitignore index 10df9072a..afe8de26a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ scripts/* !scripts/demo/ scripts/demo/* !scripts/demo/demo_markerless_calibration.py +!scripts/demo/demo_opencap_calibration.py scripts/ralph_wiggum_viz/output/ tests/debug/* *context.txt diff --git a/scripts/demo/demo_opencap_calibration.py b/scripts/demo/demo_opencap_calibration.py new file mode 100644 index 000000000..112e93dc1 --- /dev/null +++ b/scripts/demo/demo_opencap_calibration.py @@ -0,0 +1,456 @@ +"""Markerless multicamera calibration on OpenCap LabValidation data, via the caliscope public API. + +RTMPose Halpe26 keypoints from synchronized video, epipolar bootstrap and +bundle adjustment with no calibration object, GeoCalib vertical estimation, a +single camera baseline (the largest, read from the OpenCap pickle) for metric +scale, and Blender scene export. + +The five iPhone cameras share known intrinsics and known extrinsics from +OpenCap's own checkerboard calibration. This script ignores those extrinsics for +the solve and recovers them from body motion alone, then scores the markerless +result against the checkerboard reference. + +OpenCap's cameras are not hardware-synced; their "*_syncdWithMocap.avi" videos +are trimmed to the mocap timeline, not aligned camera-to-camera, and carry a +constant per-camera frame offset up to ~6 frames. The alignment stage below +measures and removes it. The default trial is STS1 (sit-to-stand): the subject +stays centered and fully framed in all five cameras, with clear vertical motion +for the sync cross-correlation to lock, slow enough that the residual sub-frame +offset is invisible. A drop jump (--trial DJ1) exposes the sub-frame floor; +overground walking cannot be synced here because the subject walks toward the +center camera, which sees looming, not the vertical bob the sync relies on. + +Inputs: + - OpenCap subject2 Session0 STS1 videos (5 cameras, ~510 frames, 60 fps, 720x1280) + - Per-camera intrinsics from each Cam{N}/cameraIntrinsicsExtrinsics.pickle + - One camera-pair distance (largest baseline), derived from the pickle extrinsics + +Outputs: + - Calibrated capture volume (camera_array.toml, image/world points) + - Blender scene with animated skeleton and camera backgrounds + - All-intra H.264 backgrounds under /videos, co-located with the .blend (self-contained snapshot) + - Procrustes and inter-camera distance comparison against the pickle reference + +Prerequisites: + - OpenCap LabValidation subject2 data (see Reproduce section below) + - RTMPose-l Halpe26 and GeoCalib weights downloaded via the caliscope GUI or CLI + +Reproduce the data: + OpenCap LabValidation is published on SimTK (Apache 2.0). From the project + page https://simtk.org/projects/opencap download the LabValidation dataset, + then point --data-dir at the subject2 folder. It must contain: + VideoData/Session0/Cam{0..4}/cameraIntrinsicsExtrinsics.pickle + VideoData/Session0/Cam{0..4}/STS1/STS1_syncdWithMocap.avi + Data credit: Uhlrich et al., 2023, PLOS Computational Biology. Subject6 opted + out and is excluded; other participants consented to identifiable video release. + +Run: + uv run python scripts/demo/demo_opencap_calibration.py \ + --data-dir /path/to/opencap/LabValidation/subject2 \ + --output-dir demo_output +""" + +import argparse +import pickle +from itertools import combinations +from pathlib import Path +from time import perf_counter + +import av +import numpy as np + +from caliscope import MODELS_DIR +from caliscope.api import ( + CameraArray, + CameraData, + CameraDistance, + calibrate_extrinsics, + estimate_vertical, + extract_image_points_multicam, + write_blender_scene, +) +from caliscope.core.point_data import ImagePoints +from caliscope.trackers import tracker_registry + +TRACKER_KEY = "ONNX_rtmpose_l_halpe26" +CAM_IDS = list(range(5)) # OpenCap Cam0..Cam4 +REF_CAM = 0 # temporal-alignment reference (OpenCap convention: first camera) + +# The pickle imageSize is stale landscape [1280, 720]; the intrinsic matrix +# (cx~360, cy~640) and the synced videos are portrait 720x1280 (width, height). +IMAGE_SIZE = (720, 1280) + + +def _cam_dir(data_dir: Path, cam_id: int, session: str = "Session0") -> Path: + return data_dir / "VideoData" / session / f"Cam{cam_id}" + + +def _load_pickles(data_dir: Path, session: str = "Session0") -> dict[int, dict]: + pickles = {} + for cam_id in CAM_IDS: + path = _cam_dir(data_dir, cam_id, session) / "cameraIntrinsicsExtrinsics.pickle" + with open(path, "rb") as f: + pickles[cam_id] = pickle.load(f) + return pickles + + +def _build_cameras(pickles: dict[int, dict]) -> CameraArray: + """One CameraData per pickle, fx and fy kept distinct (they differ slightly).""" + cameras = {} + for cam_id, d in pickles.items(): + K = np.asarray(d["intrinsicMat"], dtype=np.float64) + cameras[cam_id] = CameraData.from_intrinsics( + cam_id=cam_id, + size=IMAGE_SIZE, + fx=float(K[0, 0]), + fy=float(K[1, 1]), + cx=float(K[0, 2]), + cy=float(K[1, 2]), + distortions=np.asarray(d["distortion"], dtype=np.float64).ravel(), + ) + return CameraArray(cameras=cameras) + + +def _pickle_center_mm(d: dict) -> np.ndarray: + """Camera center in millimeters from the pickle extrinsics: C = -R.T @ t.""" + R = np.asarray(d["rotation"], dtype=np.float64) + t = np.asarray(d["translation"], dtype=np.float64).reshape(3) + return -R.T @ t + + +def _kabsch(P: np.ndarray, Q: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Rigid rotation and translation mapping P onto Q (no scaling).""" + cP, cQ = P.mean(axis=0), Q.mean(axis=0) + H = (P - cP).T @ (Q - cQ) + U, _, Vt = np.linalg.svd(H) + d = np.sign(np.linalg.det(Vt.T @ U.T)) + R = Vt.T @ np.diag([1.0, 1.0, d]) @ U.T + return R, cQ - R @ cP + + +def _transcode_all_intra( + src: Path, dst: Path, *, fps: int = 60, crf: int = 18, skip: int = 0, count: int | None = None +) -> int: + """Re-encode a video to all-intra H.264 (gop size 1), returning frames written. + + Blender scrubs non-all-intra codecs by snapping to the nearest I-frame, so + inter-coded backgrounds drift out of sync between cameras at the same scene + frame. Making every frame a keyframe keeps each camera frame-accurate under + scrubbing. Re-encoding on a fresh 60 fps timeline also drops the source AVI's + phantom timeline slots (its header counts 170 for 166 real frames). + + `skip` drops that many leading source frames and `count` caps the output + length, so the emitted file's frame index equals the aligned sync index. This + is what keeps the Blender backgrounds honest with the calibrated 3D after the + temporal-alignment stage shifts each camera onto a common timeline. + """ + dst.parent.mkdir(parents=True, exist_ok=True) + written = 0 + with av.open(str(src)) as ic, av.open(str(dst), "w") as oc: + istream = ic.streams.video[0] + ostream = oc.add_stream("libx264", rate=fps) + ostream.width = istream.codec_context.width + ostream.height = istream.codec_context.height + ostream.pix_fmt = "yuv420p" + ostream.codec_context.gop_size = 1 + ostream.options = {"crf": str(crf), "x264-params": "keyint=1:min-keyint=1:scenecut=0"} + for i, frame in enumerate(ic.decode(istream)): + if i < skip or (count is not None and written >= count): + continue + frame.pts = None + for packet in ostream.encode(frame): + oc.mux(packet) + written += 1 + for packet in ostream.encode(): + oc.mux(packet) + return written + + +# ─── SPIKE: video-only temporal alignment ──────────────────────────────────── +# OpenCap's "syncdWithMocap" videos are trimmed to the mocap window, not aligned +# camera-to-camera; a constant per-camera integer frame offset of up to ~6 frames +# remains. Left uncorrected it inflates bundle-adjustment reprojection error and +# desynchronizes the Blender backgrounds. This measures the offset from the +# tracked keypoints and shifts each camera onto a common timeline. Spike for the +# real feature (ImagePoints.temporally_aligned); see specs/video-sync-research.md. + + +def _camera_motion_signal(df, cam_id: int, sync_range: range) -> np.ndarray: + """Per-frame vertical motion signal for one camera: mean keypoint y, gap-filled.""" + sub = df[df["cam_id"] == cam_id] + y = sub.groupby("sync_index")["img_loc_y"].mean() + return y.reindex(sync_range).interpolate().bfill().ffill().to_numpy() + + +def _best_lag(ref: np.ndarray, sig: np.ndarray, max_lag: int) -> tuple[int, float]: + """Integer lag maximizing normalized cross-correlation. Positive: sig leads ref.""" + r0 = (ref - ref.mean()) / ref.std() + s0 = (sig - sig.mean()) / sig.std() + best, best_r = 0, -2.0 + for lag in range(-max_lag, max_lag + 1): + if lag >= 0: + r = float(np.corrcoef(r0[lag:], s0[: len(s0) - lag])[0, 1]) + else: + r = float(np.corrcoef(r0[:lag], s0[-lag:])[0, 1]) + if r > best_r: + best_r, best = r, lag + return best, best_r + + +def _measure_lags(image_points, ref_cam: int, max_lag: int = 10) -> dict[int, int]: + """Per-camera integer frame lag vs the reference, from keypoint cross-correlation. + + Correlation alone is ambiguous on periodic motion (thin peak margin); the + reprojection refinement below settles the exact integer. See the research note. + """ + df = image_points.df + lo, hi = int(df["sync_index"].min()), int(df["sync_index"].max()) + rng = range(lo, hi + 1) + ref_sig = _camera_motion_signal(df, ref_cam, rng) + lags = {} + for cam_id in sorted(int(c) for c in df["cam_id"].unique()): + lag, r = _best_lag(ref_sig, _camera_motion_signal(df, cam_id, rng), max_lag) + lags[cam_id] = lag + print(f" cam{cam_id}: lag {lag:+d} frames (r={r:.4f})") + return lags + + +def _apply_lags(image_points, lags: dict[int, int]): + """Shift each camera's sync_index by its lag, trim to the common window, rebase to 0.""" + df = image_points.df + df["sync_index"] = df["sync_index"] + df["cam_id"].map(lags) + lo = int(df.groupby("cam_id")["sync_index"].min().max()) + hi = int(df.groupby("cam_id")["sync_index"].max().min()) + df = df[(df["sync_index"] >= lo) & (df["sync_index"] <= hi)].copy() + df["sync_index"] = df["sync_index"] - lo + return ImagePoints(df), lo, hi + + +def _refine_lags_by_reprojection(image_points, cameras, lags: dict[int, int], ref_cam: int) -> dict[int, int]: + """Coordinate-descent on integer lags, minimizing bundle-adjustment RMSE. + + The correlation seed is ambiguous under periodic gait; reprojection error is + the honest arbiter (OpenCap does the same via triangulation). One pass over + the non-reference cameras, +/-2 frames around the seed, keeping any RMSE gain. + """ + + def rmse(candidate: dict[int, int]) -> float: + aligned, _, _ = _apply_lags(image_points, candidate) + run = calibrate_extrinsics(aligned, cameras, constraints=None, refine_intrinsics=False) + return float(run.capture_volume.reprojection_report.overall_rmse) + + best, best_rmse = dict(lags), rmse(lags) + print(f" seed RMSE {best_rmse:.2f} px") + for cam_id in sorted(lags): + if cam_id == ref_cam: + continue + for delta in (-2, -1, 1, 2): + trial = dict(best) + trial[cam_id] = lags[cam_id] + delta + r = rmse(trial) + if r < best_rmse: + best_rmse, best = r, trial + print(f" refined RMSE {best_rmse:.2f} px, lags { {c: best[c] for c in sorted(best)} }") + return best + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Markerless multicamera calibration on OpenCap LabValidation data.") + parser.add_argument( + "--data-dir", + type=Path, + required=True, + help="OpenCap LabValidation subject2 directory (contains VideoData/Session0/Cam{0..4}).", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("demo_output"), + help="Where to write the calibrated snapshot and Blender scene (default: demo_output).", + ) + parser.add_argument( + "--session", + default="Session0", + help="Session folder under VideoData/ (default: Session0). Walking trials live in Session1, " + "which has its own per-session calibration pickle.", + ) + parser.add_argument( + "--trial", + default="STS1", + help=( + "Trial name under each Cam{N}/ (default: STS1 sit-to-stand). A slow, centered trial " + "(STS1, squats1) hides the sub-frame sync residual that a drop jump (DJ1) exposes; " + "keypoint cross-correlation needs motion (static1 is too still) and consistent framing " + "in all cameras (overground walking transits out of view and cannot sync)." + ), + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + data_dir = args.data_dir.expanduser() + output_dir = args.output_dir.expanduser() + if not data_dir.exists(): + raise SystemExit(f"--data-dir not found: {data_dir}") + + t_total = perf_counter() + timings: dict[str, float] = {} + + # ── INPUTS ─────────────────────────────────────────────────────────────── + + session = args.session + pickles = _load_pickles(data_dir, session) + cameras = _build_cameras(pickles) + trial = args.trial + videos = {cam_id: _cam_dir(data_dir, cam_id, session) / trial / f"{trial}_syncdWithMocap.avi" for cam_id in CAM_IDS} + print(f"Session: {session} Trial: {trial}") + + # Largest camera baseline from the pickle extrinsics, the most stable scale cue. + pickle_centers_mm = {cam_id: _pickle_center_mm(pickles[cam_id]) for cam_id in CAM_IDS} + scale_a, scale_b = max( + combinations(CAM_IDS, 2), + key=lambda pair: np.linalg.norm(pickle_centers_mm[pair[0]] - pickle_centers_mm[pair[1]]), + ) + scale_m = float(np.linalg.norm(pickle_centers_mm[scale_a] - pickle_centers_mm[scale_b]) / 1000.0) + scale_cue = CameraDistance(cam_a=scale_a, cam_b=scale_b, meters=scale_m) + print(f"Scale cue: cam{scale_a}-cam{scale_b} baseline {scale_m:.4f} m (largest, from pickle extrinsics)") + + # ── 1. KEYPOINTS ───────────────────────────────────────────────────────── + + print("\nExtracting keypoints...") + tracker_registry.scan_onnx_models(MODELS_DIR) + tracker = tracker_registry.create(TRACKER_KEY) + t0 = perf_counter() + image_points = extract_image_points_multicam(videos, tracker, frame_step=1) + timings["extraction"] = perf_counter() - t0 + print(f" {len(image_points.df)} observations in {timings['extraction']:.1f}s") + + # ── 1b. TEMPORAL ALIGNMENT ─────────────────────────────────────────────── + # OpenCap's "syncdWithMocap" videos carry a constant per-camera frame offset. + # Measure it from the keypoints, settle the integer by reprojection, and shift + # every camera onto a common timeline before calibrating. + + print("\nMeasuring per-camera frame offsets...") + t0 = perf_counter() + seed_lags = _measure_lags(image_points, ref_cam=REF_CAM) + lags = _refine_lags_by_reprojection(image_points, cameras, seed_lags, ref_cam=REF_CAM) + image_points, window_lo, window_hi = _apply_lags(image_points, lags) + n_aligned = window_hi - window_lo + 1 + timings["alignment"] = perf_counter() - t0 + print(f" aligned to {n_aligned} common frames in {timings['alignment']:.1f}s") + + # ── 2. EXTRINSICS ──────────────────────────────────────────────────────── + + print("\nCalibrating extrinsics...") + t0 = perf_counter() + run = calibrate_extrinsics(image_points, cameras, constraints=None, refine_intrinsics=False) + timings["extrinsics"] = perf_counter() - t0 + volume = run.capture_volume + print(f" reprojection RMSE {volume.reprojection_report.overall_rmse:.2f} px in {timings['extrinsics']:.1f}s") + + # ── 3. ANCHORING ───────────────────────────────────────────────────────── + + print("\nEstimating vertical (GeoCalib)...") + t0 = perf_counter() + vertical = estimate_vertical(videos, cameras, frames_per_camera=1) + timings["vertical"] = perf_counter() - t0 + print(f" done in {timings['vertical']:.1f}s") + + print("\nAnchoring (orient, scale, ground, center)...") + t0 = perf_counter() + volume = volume.oriented(up=vertical.up_per_cam) + volume = volume.scaled(scale_cue) + volume = volume.grounded() + volume = volume.centered() + timings["anchoring"] = perf_counter() - t0 + print(f" done in {timings['anchoring']:.2f}s") + + # ── 4. VALIDATION VS OPENCAP PICKLE ────────────────────────────────────── + + print("\nComparing against the OpenCap pickle calibration...") + cam_ids = sorted(volume.camera_array.posed_cameras) + cali_centers_mm = { + cam_id: (-cam.rotation.T @ cam.translation) * 1000.0 + for cam_id, cam in volume.camera_array.posed_cameras.items() + } + + P = np.array([cali_centers_mm[c] for c in cam_ids]) + Q = np.array([pickle_centers_mm[c] for c in cam_ids]) + R_align, T_align = _kabsch(P, Q) + P_aligned = (R_align @ P.T).T + T_align + + print(" per-camera pose error after rigid Procrustes (no scaling):") + for i, c in enumerate(cam_ids): + center_err_mm = float(np.linalg.norm(P_aligned[i] - Q[i])) + R_cali = np.asarray(volume.camera_array.cameras[c].rotation, dtype=np.float64) + R_ref = np.asarray(pickles[c]["rotation"], dtype=np.float64) + R_err = R_ref @ (R_cali @ R_align.T).T + rot_err_deg = float(np.degrees(np.arccos(np.clip((np.trace(R_err) - 1.0) / 2.0, -1.0, 1.0)))) + print(f" cam{c}: center {center_err_mm:6.1f} mm, rotation {rot_err_deg:5.2f} deg") + + print(" inter-camera distance (caliscope vs pickle):") + errors_mm = [] + for cam_a, cam_b in combinations(cam_ids, 2): + cali_mm = float(np.linalg.norm(cali_centers_mm[cam_a] - cali_centers_mm[cam_b])) + ref_mm = float(np.linalg.norm(pickle_centers_mm[cam_a] - pickle_centers_mm[cam_b])) + err_mm = cali_mm - ref_mm + err_pct = 100.0 * err_mm / ref_mm + tag = " (scale reference)" if (cam_a, cam_b) == (scale_a, scale_b) else "" + errors_mm.append(err_mm) + print( + f" cam{cam_a}-cam{cam_b}: caliscope {cali_mm:7.1f} mm, " + f"pickle {ref_mm:7.1f} mm, error {err_mm:+6.1f} mm ({err_pct:+.2f}%){tag}" + ) + rmse_mm = float(np.sqrt(np.mean(np.square(errors_mm)))) + print(f" RMSE vs pickle: {rmse_mm:.1f} mm over {len(errors_mm)} pairs") + print(" OpenCap checkerboard benchmark: sub-cm, sub-0.25-degree.") + + # ── 5. BLENDER EXPORT ──────────────────────────────────────────────────── + + print("\nTriangulating world points...") + t0 = perf_counter() + world_points = image_points.triangulate(volume.camera_array) + timings["triangulation"] = perf_counter() - t0 + print(f" {len(world_points.df)} points in {timings['triangulation']:.1f}s") + + # Backgrounds only; calibration runs on the source AVIs above. Each camera + # skips window_lo - lag leading frames so the emitted frame index equals the + # aligned sync index, keeping the Blender backgrounds honest with the 3D. + print("\nTranscoding backgrounds (all-intra H.264, temporally aligned)...") + t0 = perf_counter() + background_videos = {} + for cam_id in CAM_IDS: + dst = output_dir / "videos" / f"cam_{cam_id}.mp4" + n_frames = _transcode_all_intra(videos[cam_id], dst, skip=window_lo - lags[cam_id], count=n_aligned) + background_videos[cam_id] = dst + print(f" cam{cam_id}: {n_frames} frames -> {dst.name}") + timings["transcode"] = perf_counter() - t0 + print(f" done in {timings['transcode']:.1f}s") + + print("\nExporting Blender scene...") + t0 = perf_counter() + volume.save(output_dir) + scene_script = write_blender_scene( + volume.camera_array, + world_points, + output_dir / "capture_volume_scene.py", + fps=60, # OpenCap iPhone footage; must match the transcoded backgrounds + videos=background_videos, + wireframe=tracker_registry.wireframe_for(TRACKER_KEY), + ) + timings["blender"] = perf_counter() - t0 + print(f" {scene_script.with_suffix('.blend')} in {timings['blender']:.1f}s") + + # ── SUMMARY ────────────────────────────────────────────────────────────── + + timings["total"] = perf_counter() - t_total + print("\nTiming summary:") + for stage, seconds in timings.items(): + print(f" {stage:<15} {seconds:6.1f}s") + + print(f"\nOpen the scene:\n blender {scene_script.with_suffix('.blend')}") + + +if __name__ == "__main__": + main() From 4826b5f5493f37716f3f6933d66d38f912c4e92a Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 11:54:53 -0500 Subject: [PATCH 03/11] feat: let the scripting API set per-camera rotation extract_image_points held rotation_count as a local pinned to 0 and the multicam path passed a literal 0, so a script had no way to say what the GUI's rotate buttons say. Both now take the quarter-turn count. Charuco, ArUco, and chessboard detection is rotation-invariant and ignores it; ONNX pose trackers need it to see an upright body. --- src/caliscope/api.py | 16 ++++++++++++-- tests/test_api.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/caliscope/api.py b/src/caliscope/api.py index 7c1306637..c6634576d 100644 --- a/src/caliscope/api.py +++ b/src/caliscope/api.py @@ -137,6 +137,7 @@ def extract_image_points( tracker: Tracker, *, frame_step: int = 1, + rotation_count: int = 0, progress: ProgressCallback | None = _AUTO, ) -> ImagePoints: """Extract 2D landmark observations from a single camera video. @@ -151,6 +152,13 @@ def extract_image_points( frame_step: Process every Nth frame (default 1 = every frame). For intrinsic calibration, frame_step=5 is typical since only ~30 diverse frames are needed. + rotation_count: Quarter turns applied before detection, matching + ``CameraData.rotation_count`` (1 = 90 degrees clockwise, negative + for counter-clockwise). Returned image coordinates are always in + the original frame, so calibration is unaffected by the choice. + Charuco, ArUco, and chessboard trackers ignore this because their + detectors are already rotation-invariant; ONNX pose trackers use + it to put an upright body in front of a model trained on one. progress: Callback invoked per-frame for progress reporting. Defaults to a Rich progress bar. Pass ``None`` to suppress output. @@ -173,7 +181,6 @@ def extract_image_points( with _auto_progress(progress) as progress: all_rows: list[dict] = [] - rotation_count = 0 props = read_video_properties(video_path) frame_count = props["frame_count"] @@ -246,6 +253,7 @@ def extract_image_points_multicam( *, frame_step: int = 1, timestamps: Path | str | None = None, + rotation_counts: Mapping[int, int] | None = None, progress: ProgressCallback | None = _AUTO, ) -> ImagePoints: """Extract synchronized 2D landmark observations from multiple camera videos. @@ -272,6 +280,9 @@ def extract_image_points_multicam( If omitted, timestamps are inferred from video metadata (FPS and frame count). Providing a CSV is recommended for recordings where cameras did not start at exactly the same time. + rotation_counts: Optional cam_id to quarter-turn mapping, matching + ``CameraData.rotation_count``. Cameras left out default to 0. See + ``extract_image_points`` for what rotation does and does not change. progress: Callback invoked per-frame for progress reporting. Defaults to a Rich progress bar. Pass ``None`` to suppress output. A single instance is shared across all camera threads (safe because @@ -300,6 +311,7 @@ def extract_image_points_multicam( # Normalize all video paths upfront video_paths: dict[int, Path] = {cam_id: Path(p) for cam_id, p in videos.items()} + rotations: Mapping[int, int] = rotation_counts or {} # Validate all video paths at once missing = {cam_id: str(p) for cam_id, p in video_paths.items() if not p.exists()} @@ -354,7 +366,7 @@ def _process_camera(cam_id: int, work_list: list[tuple[int, int]], video_path: P rows: list[dict] = [] processed = 0 while (raw := frame_source.next_frame()) is not None: - point_packet = tracker.get_points(raw.frame, cam_id=cam_id, rotation_count=0) + point_packet = tracker.get_points(raw.frame, cam_id=cam_id, rotation_count=rotations.get(cam_id, 0)) n_points = len(point_packet.keypoint_id) sync_index = sync_for[raw.frame_index] frame_time = synced.time_for(cam_id, raw.frame_index) diff --git a/tests/test_api.py b/tests/test_api.py index 8c462b8e7..b744ff367 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -344,6 +344,50 @@ def test_extract_image_points_multicam_pipeline(tmp_path: Path): assert rmse < 2.0, f"Reprojection RMSE {rmse:.4f} px exceeds 2.0 px threshold" +# --------------------------------------------------------------------------- +# rotation_count plumbing +# --------------------------------------------------------------------------- + + +class _RotationSpyTracker(CharucoTracker): + """Charuco tracker that records the rotation_count it was handed.""" + + def __init__(self, charuco: Charuco) -> None: + super().__init__(charuco) + self.seen: list[tuple[int, int]] = [] + + def _detect(self, frame, cam_id: int = 0, rotation_count: int = 0): + self.seen.append((cam_id, rotation_count)) + return super()._detect(frame, cam_id, rotation_count) + + +def test_extract_image_points_forwards_rotation_count(): + """The GUI sets a per-camera rotation; the API must be able to say the same thing.""" + charuco = Charuco.from_toml(PRERECORDED_SESSION / "charuco.toml") + tracker = _RotationSpyTracker(charuco) + + video_path = PRERECORDED_SESSION / "calibration" / "intrinsic" / "cam_0.mp4" + extract_image_points(video_path, 0, tracker, frame_step=20, rotation_count=1, progress=None) + + assert tracker.seen + assert {rotation for _, rotation in tracker.seen} == {1} + + +def test_extract_image_points_multicam_forwards_rotation_counts(tmp_path: Path): + """Each camera gets its own rotation; cameras left out of the mapping get zero.""" + copy_contents_to_clean_dest(CHARUCO_SESSION, tmp_path) + + extrinsic_dir = tmp_path / "calibration" / "extrinsic" + videos = {cam_id: extrinsic_dir / f"cam_{cam_id}.mp4" for cam_id in (0, 1)} + + charuco = Charuco.from_toml(tmp_path / "charuco.toml") + tracker = _RotationSpyTracker(charuco) + + extract_image_points_multicam(videos, tracker, frame_step=30, rotation_counts={1: -1}, progress=None) + + assert set(tracker.seen) == {(0, 0), (1, -1)} + + # --------------------------------------------------------------------------- # Debug harness # --------------------------------------------------------------------------- @@ -397,4 +441,11 @@ def test_extract_image_points_multicam_pipeline(tmp_path: Path): with tempfile.TemporaryDirectory() as tmp: test_extract_image_points_multicam_pipeline(Path(tmp)) + logger.info("test_extract_image_points_forwards_rotation_count") + test_extract_image_points_forwards_rotation_count() + + logger.info("test_extract_image_points_multicam_forwards_rotation_counts") + with tempfile.TemporaryDirectory() as tmp: + test_extract_image_points_multicam_forwards_rotation_counts(Path(tmp)) + logger.info("All API tests passed.") From 686c322ba76e11b4e546558b8bd70f11c4e33ef9 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 11:54:58 -0500 Subject: [PATCH 04/11] fix: accept str paths across the documented persistence surface CaptureVolume.save and load take Path | str, but CameraArray.from_toml, CameraArray.to_toml, to_aniposelib_toml, and the Charuco TOML methods required a Path and raised AttributeError on a string. The scripting docs pass strings, so the reload example in them could not run. Also pins the board-size equivalence the docs now promise: board_width and board_height never reach the geometry, because both the GUI and from_squares set square_size_override_cm. --- src/caliscope/cameras/camera_array.py | 9 ++++++--- src/caliscope/core/charuco.py | 14 ++++++++------ tests/test_charuco.py | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/caliscope/cameras/camera_array.py b/src/caliscope/cameras/camera_array.py index 303bfd9ac..05dee06f8 100644 --- a/src/caliscope/cameras/camera_array.py +++ b/src/caliscope/cameras/camera_array.py @@ -375,7 +375,7 @@ def normalized_projection_matrices(self) -> dict[int, np.ndarray]: return proj_mat @classmethod - def from_toml(cls, path: Path) -> "CameraArray": + def from_toml(cls, path: Path | str) -> "CameraArray": """Load CameraArray from TOML file. Rotation is stored as a 3x1 Rodrigues vector in TOML but may be a 3x3 @@ -388,6 +388,7 @@ def from_toml(cls, path: Path) -> "CameraArray": from caliscope.persistence import PersistenceError from caliscope.core.toml_helpers import _list_to_array, _clean_scalar + path = Path(path) if not path.exists(): raise PersistenceError(f"CameraArray file not found: {path}") @@ -440,7 +441,7 @@ def from_toml(cls, path: Path) -> "CameraArray": return cls(cameras_dict) - def to_toml(self, path: Path) -> None: + def to_toml(self, path: Path | str) -> None: """Save CameraArray to TOML file. Converts 3x3 rotation matrices to 3x1 Rodrigues vectors for storage. @@ -451,6 +452,7 @@ def to_toml(self, path: Path) -> None: from caliscope.persistence import PersistenceError, _safe_write_toml from caliscope.core.toml_helpers import _array_to_list + path = Path(path) try: path.parent.mkdir(parents=True, exist_ok=True) @@ -486,7 +488,7 @@ def to_toml(self, path: Path) -> None: except Exception as e: raise PersistenceError(f"Failed to save CameraArray to {path}: {e}") from e - def to_aniposelib_toml(self, path: Path) -> None: + def to_aniposelib_toml(self, path: Path | str) -> None: """Save CameraArray in aniposelib-compatible TOML format. Only exports posed cameras. Uses top-level [cam_N] sections instead of @@ -498,6 +500,7 @@ def to_aniposelib_toml(self, path: Path) -> None: from caliscope.persistence import PersistenceError, _safe_write_toml from caliscope.core.toml_helpers import _array_to_list + path = Path(path) try: path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/caliscope/core/charuco.py b/src/caliscope/core/charuco.py index a98f6cf94..b2686e156 100644 --- a/src/caliscope/core/charuco.py +++ b/src/caliscope/core/charuco.py @@ -272,18 +272,18 @@ def board_img(self, pixmap_scale=1000): return img - def save_image(self, path): + def save_image(self, path: Path | str) -> None: """ Saving image at 10x higher resolution than used for GUI """ - cv2.imwrite(path, self.board_img(pixmap_scale=10000)) + cv2.imwrite(str(path), self.board_img(pixmap_scale=10000)) - def save_mirror_image(self, path): + def save_mirror_image(self, path: Path | str) -> None: """ Saving image at 10x higher resolution than used for GUI """ mirror = cv2.flip(self.board_img(pixmap_scale=10000), 1) - cv2.imwrite(path, mirror) + cv2.imwrite(str(path), mirror) def get_connected_points(self) -> set[tuple[int, int]]: """ @@ -353,7 +353,7 @@ def _fit_dictionary(self) -> None: self.dictionary = fit_dictionary_pool(self.dictionary, self.marker_count) @classmethod - def from_toml(cls, path: Path) -> "Charuco": + def from_toml(cls, path: Path | str) -> "Charuco": """Load Charuco board definition from TOML file. Normalizes the dictionary pool to fit the board (see fit_dictionary_pool), @@ -366,6 +366,7 @@ def from_toml(cls, path: Path) -> "Charuco": """ from caliscope.persistence import PersistenceError + path = Path(path) if not path.exists(): raise PersistenceError(f"Charuco file not found: {path}") @@ -378,7 +379,7 @@ def from_toml(cls, path: Path) -> "Charuco": charuco._fit_dictionary() return charuco - def to_toml(self, path: Path) -> None: + def to_toml(self, path: Path | str) -> None: """Save Charuco board definition to TOML file. Enumerates fields explicitly rather than using __dict__ to avoid @@ -395,6 +396,7 @@ def to_toml(self, path: Path) -> None: """ from caliscope.persistence import PersistenceError, _safe_write_toml + path = Path(path) self._fit_dictionary() try: path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_charuco.py b/tests/test_charuco.py index fdeaf7f6b..22dd402d3 100644 --- a/tests/test_charuco.py +++ b/tests/test_charuco.py @@ -165,6 +165,30 @@ def test_negative_thickness_rejected(): Charuco.from_squares(columns=4, rows=5, square_size_cm=5.0, thickness_cm=-0.5) +# -- Board dimensions are a printing concern, not a calibration one ----------- + + +def test_board_dimensions_do_not_change_geometry(): + """A scripted board and a GUI-configured board calibrate identically. + + The GUI collects board_width, board_height, and units; from_squares does not. + Those dimensions only reach the geometry through the fallback square length, + which never fires because both paths set square_size_override_cm. Scripting + docs make this promise to users, so pin it. + """ + scripted = Charuco.from_squares(columns=4, rows=5, square_size_cm=5.4) + gui_configured = Charuco( + columns=4, + rows=5, + board_height=11.0, + board_width=8.5, + units="inch", + square_size_override_cm=5.4, + ) + + assert scripted.board.getChessboardCorners() == pytest.approx(gui_configured.board.getChessboardCorners()) + + if __name__ == "__main__": debug_dir = Path(__file__).parent / "tmp" debug_dir.mkdir(parents=True, exist_ok=True) From 326f76fe3a85a2560bfa83031ea9bb7e9853d76a Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 11:55:05 -0500 Subject: [PATCH 05/11] docs: cover board printing, axis rotation, and aniposelib export in the scripting guide The guide stopped at align_to_object and save, so three things the GUI does had no scripted equivalent on the page: printing a board, rotating the axes, and writing camera_array_aniposelib.toml. All three already existed in the API. Adds a GUI control to API call table so the gap is visible at a glance. Also states what the board width and height do, since the GUI collects them and from_squares does not: they set the printed aspect ratio and nothing else. --- docs/scripting.md | 99 ++++++++++++++++++++++++++++++++++++++++++++- scripts/demo_api.py | 18 +++++++-- 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/docs/scripting.md b/docs/scripting.md index da6da8608..7ab992a3f 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -49,6 +49,29 @@ For a two-sided board on a real substrate, pass the measured `thickness_cm` so b charuco = Charuco.from_squares(columns=4, rows=5, square_size_cm=3.0, thickness_cm=0.6) ``` +### Printing the board + +The GUI's board panel also collects a board width, height, and unit. +Those dimensions never reach the calibration. +Both the GUI and `from_squares` pin the geometry to the measured square size, so the width and height set only the aspect ratio of the printed image. +A board built by `from_squares` and the same board configured in the GUI produce identical corner positions. + +Use the full constructor when you want a printable board sized to your paper: + +```python +charuco = Charuco( + columns=4, rows=5, + board_width=8.5, board_height=11.0, units="inch", + square_size_override_cm=5.4, +) +charuco.save_image("charuco.png") +charuco.save_mirror_image("charuco_mirror.png") +``` + +Print it, measure a square with calipers, and pass the measurement as `square_size_cm` or `square_size_override_cm`. +Printers rescale. +The mirror image is the back face of a two-sided board. + ## Step 2: Build a camera array from video metadata ```python @@ -113,6 +136,17 @@ ext_points = extract_image_points_multicam(extrinsic_videos, tracker, timestamps `frame_step` works on time-aligned moments, not raw frames. `frame_step=10` processes every 10th synchronized moment. +If a camera recorded sideways, give it a quarter-turn count, the same value the GUI's rotate buttons set: + +```python +ext_points = extract_image_points_multicam(extrinsic_videos, tracker, rotation_counts={2: 1, 3: -1}) +``` + +Rotation decides only what the tracker sees. +Detected points come back in original image coordinates either way, so calibration results do not change. +Charuco, ArUco, and chessboard detection is already rotation-invariant and ignores the setting. +ONNX pose trackers need it because they were trained on upright people. + ### Check camera pair coverage Before calibrating, verify that camera pairs share enough observations: @@ -185,6 +219,35 @@ volume = volume.align_to_object(sync_idx) This applies a similarity transform (Umeyama algorithm) that aligns the world coordinate frame to the board's position at the chosen sync index. Pick a frame where the board is at your desired origin. +### Rotate the axes + +The board rarely lands with its axes where your lab convention wants them. +`rotate` turns the whole volume, cameras and points together. +It is what the GUI's X, Y, and Z buttons call: + +```python +volume = volume.rotate("x", 90) +volume = volume.rotate("z", -90) +``` + +Positive angles follow the right-hand rule: counter-clockwise looking down the positive axis toward the origin. +The GUI buttons are fixed at 90 degrees, but any angle works here. + +### Anchoring without a board + +A markerless calibration has no board to align to, so the frame comes from the scene instead: + +```python +volume = volume.oriented(up=vertical.up_per_cam) # +Z becomes vertical +volume = volume.scaled(CameraDistance(cam_a=0, cam_b=3, meters=4.2)) +volume = volume.grounded() # floor at Z=0 +volume = volume.centered() # XY origin at the camera centroid +``` + +Call them in that order. +`grounded` assumes Z is already vertical, and `centered` assumes the floor is already at zero. +`oriented` takes a per-camera up vector, which `estimate_vertical` produces from the video. + ## Step 7: Inspect results ```python @@ -199,7 +262,7 @@ The report shows optimization status, reprojection error percentiles, per-camera volume.save("capture_volume") ``` -This writes three files to the directory: `camera_array.toml`, `image_points.csv`, and `world_points.csv`. +This writes `camera_array.toml`, `image_points.csv`, and `world_points.csv` to the directory, plus `constraints.toml` when the volume carries constraints. To reload later: ```python @@ -207,6 +270,20 @@ volume = CaptureVolume.load("capture_volume") cameras = CameraArray.from_toml("capture_volume/camera_array.toml") ``` +### Aniposelib export + +Pose2Sim, anipose, and other aniposelib-compatible tools want their own TOML layout. +The GUI writes one to the workspace root every time it saves. +A script asks for it: + +```python +volume.camera_array.to_aniposelib_toml("camera_array_aniposelib.toml") +``` + +Only posed cameras are written, with rotations as Rodrigues vectors. +`save` leaves this out on purpose. +Its directory stays a self-contained snapshot of the calibration, not a mix of snapshot and handoff files. + ## Calibrating without intrinsics (experimental) !!! warning "Experimental" @@ -261,3 +338,23 @@ Detection is all-or-nothing, so a frame where any corner is cut off or covered c A board with both counts even or both odd looks identical after a half turn, so the detector cannot tell the two orientations apart. When two cameras see opposite orientations, corner ids reverse and triangulation silently pairs the wrong points. A ChArUco or ArUco target does not have this problem. + +## Matching the GUI + +The two surfaces share their calibration code, so the same inputs give the same numbers. +The GUI just does more of the surrounding steps for you. + +| GUI control | API equivalent | +|---|---| +| Board shape and square size | `Charuco.from_squares(columns, rows, square_size_cm)` | +| Board size and units | Print-only. Use the full `Charuco(...)` constructor. | +| Invert | `Charuco.from_squares(..., inverted=True)` | +| Save board PNG, mirror PNG | `charuco.save_image(path)`, `charuco.save_mirror_image(path)` | +| Camera rotate buttons | `rotation_counts={cam_id: turns}` at extraction | +| Calibrate | `calibrate_extrinsics(points, cameras, constraints)` | +| Refine intrinsics checkbox | `refine_intrinsics=True` | +| Filter outliers | `filter_percentile=2.5` | +| Set origin at frame | `volume.align_to_object(sync_index)` | +| X, Y, Z rotate buttons | `volume.rotate(axis, degrees)` | +| Saving a calibration | `volume.save(directory)` | +| `camera_array_aniposelib.toml` | `volume.camera_array.to_aniposelib_toml(path)` | diff --git a/scripts/demo_api.py b/scripts/demo_api.py index 6ee779f06..588fcafeb 100644 --- a/scripts/demo_api.py +++ b/scripts/demo_api.py @@ -6,8 +6,8 @@ 3. Calibrate intrinsics (per camera) 4. Extract time-aligned extrinsic points 5. Bootstrap + optimize capture volume - 6. Align to object coordinates - 7. Save and reload results + 6. Align to object coordinates and rotate the axes + 7. Save, export for aniposelib, and reload results Usage: uv run python scripts/demo_api.py @@ -99,18 +99,28 @@ volume = volume.optimize(strict=False) print(f" Pass 2 RMSE: {volume.reprojection_report.overall_rmse:.3f} px") -# --- 6. Align to object coordinates --- +# --- 6. Align to object coordinates, then rotate the axes --- print("\nStep 6: Align to object coordinates") sync_idx = volume.unique_sync_indices[len(volume.unique_sync_indices) // 2] volume = volume.align_to_object(int(sync_idx)) +# Same transform as the GUI's X/Y/Z buttons: the board defines the origin, but +# its axes rarely match the lab convention you want to hand downstream. +volume = volume.rotate("x", 90) +print(" Rotated +90 degrees about X") + print_extrinsic_report(volume) -# --- 7. Save and reload --- +# --- 7. Save, export for aniposelib, and reload --- print("\nStep 7: Save and reload") volume.save(OUTPUT_DIR) print(f" Saved capture volume to {OUTPUT_DIR}") +# The GUI writes this to the workspace root on every save; a script asks for it. +aniposelib_path = OUTPUT_DIR / "camera_array_aniposelib.toml" +volume.camera_array.to_aniposelib_toml(aniposelib_path) +print(f" Exported aniposelib camera array to {aniposelib_path}") + # Verify round-trip volume_reloaded = CaptureVolume.load(OUTPUT_DIR) cameras_reloaded = CameraArray.from_toml(OUTPUT_DIR / "camera_array.toml") From 41687c77a53eccafa5dfce719bb15d5c3126bad0 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 13:31:20 -0500 Subject: [PATCH 06/11] feat: add CaptureVolume.translate for fixed coordinate shifts grounded() rests the lowest triangulated point on Z=0, but that point is a marker rather than the floor. A toe marker rides above the ground on the shoe and its own thickness, so the recovered floor sits low by that amount. There was no way to correct it short of hand-editing the saved points. translate shifts points and cameras together by a fixed offset in meters, leaving shape and scale alone. It is the companion to rotate. --- docs/scripting.md | 20 ++++++++++++ src/caliscope/core/capture_volume.py | 37 ++++++++++++++++++++++ tests/test_capture_volume_anchoring.py | 43 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/docs/scripting.md b/docs/scripting.md index 7ab992a3f..d89c9c6da 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -233,6 +233,17 @@ volume = volume.rotate("z", -90) Positive angles follow the right-hand rule: counter-clockwise looking down the positive axis toward the origin. The GUI buttons are fixed at 90 degrees, but any angle works here. +### Shift the origin + +`translate` moves the whole volume by a fixed offset in meters: + +```python +volume = volume.translate(z=0.02) +``` + +Shape and scale do not change, only where the rig sits in world coordinates. +Positive z raises everything. + ### Anchoring without a board A markerless calibration has no board to align to, so the frame comes from the scene instead: @@ -248,6 +259,14 @@ Call them in that order. `grounded` assumes Z is already vertical, and `centered` assumes the floor is already at zero. `oriented` takes a per-camera up vector, which `estimate_vertical` produces from the video. +`grounded` rests the lowest triangulated point on Z=0, and that point is a marker, not the floor. +A toe marker rides above the ground on the shoe and its own thickness. +Measure that height and lift the volume by it: + +```python +volume = volume.grounded().translate(z=0.02) +``` + ## Step 7: Inspect results ```python @@ -356,5 +375,6 @@ The GUI just does more of the surrounding steps for you. | Filter outliers | `filter_percentile=2.5` | | Set origin at frame | `volume.align_to_object(sync_index)` | | X, Y, Z rotate buttons | `volume.rotate(axis, degrees)` | +| No GUI equivalent | `volume.translate(x, y, z)` | | Saving a calibration | `volume.save(directory)` | | `camera_array_aniposelib.toml` | `volume.camera_array.to_aniposelib_toml(path)` | diff --git a/src/caliscope/core/capture_volume.py b/src/caliscope/core/capture_volume.py index 291fa25ba..b968dde3f 100644 --- a/src/caliscope/core/capture_volume.py +++ b/src/caliscope/core/capture_volume.py @@ -1007,6 +1007,43 @@ def rotate(self, axis: Literal["x", "y", "z"], angle_degrees: float) -> "Capture _optimization_status=self._optimization_status, ) + def translate(self, x: float = 0.0, y: float = 0.0, z: float = 0.0) -> "CaptureVolume": + """Shift the coordinate system by a fixed offset in meters. + + Adds the offset to every world point and carries the cameras with them. + The rig's shape and scale are untouched; only where it sits in world + coordinates moves. Positive z raises everything. + + The companion to ``rotate``, and the correction ``grounded`` cannot make + for you. ``grounded`` drops the lowest triangulated point to Z=0, but + that point is a marker, not the floor: a toe marker rides above the + ground on the shoe and its own thickness. Lift the volume by that + measured height to put the real floor at zero. + + Args: + x: Offset along the world X axis, in meters. + y: Offset along the world Y axis, in meters. + z: Offset along the world Z axis, in meters. + + Returns: + New CaptureVolume with the shifted coordinate system. + """ + transform = SimilarityTransform( + rotation=np.eye(3, dtype=np.float64), + translation=np.array([x, y, z], dtype=np.float64), + scale=1.0, + ) + + new_camera_array, new_world_points = apply_similarity_transform(self.camera_array, self.world_points, transform) + + return CaptureVolume( + camera_array=new_camera_array, + image_points=self.image_points, + world_points=new_world_points, + constraints=self.constraints, + _optimization_status=self._optimization_status, + ) + # ------------------------------------------------------------------ # Metric anchoring (post-BA). Each returns a new frozen CaptureVolume. # ------------------------------------------------------------------ diff --git a/tests/test_capture_volume_anchoring.py b/tests/test_capture_volume_anchoring.py index 4fa2d8b92..fda2573db 100644 --- a/tests/test_capture_volume_anchoring.py +++ b/tests/test_capture_volume_anchoring.py @@ -431,6 +431,47 @@ def test_grounded_rejects_unknown_mode(): volume.grounded("centroid") # type: ignore[arg-type] +# --------------------------------------------------------------------------- +# Translation +# --------------------------------------------------------------------------- + + +def test_translate_lifts_floor_by_marker_height(): + """grounded() rests the lowest marker at zero; translate corrects for its height. + + A toe marker rides above the ground on the shoe, so the real floor sits + below it. Lifting by the measured height puts the floor at Z=0 and leaves + the marker where it belongs, above it. + """ + cameras = { + 0: make_camera(0, np.eye(3), np.array([5.0, 7.0, 3.0])), + 1: make_camera(1, np.eye(3), np.array([8.0, 7.0, 3.0])), + } + rows = [world_row(0, 10, (1.0, 1.0, 2.0)), world_row(0, 11, (1.0, 1.0, 4.0))] + grounded = make_volume(cameras, rows).grounded("lowest_point") + + lifted = grounded.translate(z=0.02) + + assert float(lifted.world_points.df["z_coord"].min()) == pytest.approx(0.02) + assert camera_center(lifted, 0) == pytest.approx(camera_center(grounded, 0) + np.array([0.0, 0.0, 0.02])) + + +def test_translate_preserves_shape_and_defaults_to_no_op(): + cameras = { + 0: make_camera(0, np.eye(3), np.array([0.0, 0.0, 0.0])), + 1: make_camera(1, np.eye(3), np.array([3.0, 0.0, 0.0])), + } + rows = [world_row(0, 10, (1.0, 1.0, 1.0)), world_row(0, 11, (2.0, 0.0, 1.5))] + volume = make_volume(cameras, rows) + + shifted = volume.translate(x=1.5, y=-0.25, z=0.02) + + # Every point moves by the same offset, so relative geometry is untouched. + delta = shifted.world_points.points - volume.world_points.points + assert delta == pytest.approx(np.tile([1.5, -0.25, 0.02], (len(delta), 1))) + assert volume.translate().world_points.points == pytest.approx(volume.world_points.points) + + # --------------------------------------------------------------------------- # Chain order invariance # --------------------------------------------------------------------------- @@ -454,6 +495,8 @@ def test_chain_order_invariance_scale_and_orient(): debug_dir = Path(__file__).parent / "tmp" debug_dir.mkdir(parents=True, exist_ok=True) + test_translate_lifts_floor_by_marker_height() + test_translate_preserves_shape_and_defaults_to_no_op() test_single_camera_distance_recovers_scale() test_single_segment_length_recovers_scale() test_single_depth_observation_recovers_scale() From f8f3897063193fe81d22f1f27761bcbdb946fc97 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 13:44:08 -0500 Subject: [PATCH 07/11] refactor: build grounded and centered on translate, add floor height correction Three copies of the same identity-rotation SimilarityTransform block existed; grounded and centered now resolve to translate instead. grounded gains lowest_point_height_m. It defaults to 0.0, so existing callers are unchanged, and the demo scripts still ground the same way. --- docs/scripting.md | 11 +++-- src/caliscope/core/capture_volume.py | 59 +++++++++++--------------- tests/test_capture_volume_anchoring.py | 25 +++++++++++ 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/docs/scripting.md b/docs/scripting.md index d89c9c6da..e88782f86 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -243,6 +243,7 @@ volume = volume.translate(z=0.02) Shape and scale do not change, only where the rig sits in world coordinates. Positive z raises everything. +This is the primitive `grounded` and `centered` are built on, exposed for the shifts they do not cover. ### Anchoring without a board @@ -259,14 +260,16 @@ Call them in that order. `grounded` assumes Z is already vertical, and `centered` assumes the floor is already at zero. `oriented` takes a per-camera up vector, which `estimate_vertical` produces from the video. -`grounded` rests the lowest triangulated point on Z=0, and that point is a marker, not the floor. -A toe marker rides above the ground on the shoe and its own thickness. -Measure that height and lift the volume by it: +By default `grounded` rests the lowest triangulated point on Z=0, and that point is a marker, not the floor. +A toe marker rides above the ground on the shoe and its own thickness, so the floor ends up buried. +Measure that height and hand it over: ```python -volume = volume.grounded().translate(z=0.02) +volume = volume.grounded(lowest_point_height_m=0.02) ``` +The marker then sits at 0.02 and the floor it stands on is zero. + ## Step 7: Inspect results ```python diff --git a/src/caliscope/core/capture_volume.py b/src/caliscope/core/capture_volume.py index b968dde3f..bc1329591 100644 --- a/src/caliscope/core/capture_volume.py +++ b/src/caliscope/core/capture_volume.py @@ -1014,11 +1014,10 @@ def translate(self, x: float = 0.0, y: float = 0.0, z: float = 0.0) -> "CaptureV The rig's shape and scale are untouched; only where it sits in world coordinates moves. Positive z raises everything. - The companion to ``rotate``, and the correction ``grounded`` cannot make - for you. ``grounded`` drops the lowest triangulated point to Z=0, but - that point is a marker, not the floor: a toe marker rides above the - ground on the shoe and its own thickness. Lift the volume by that - measured height to put the real floor at zero. + The companion to ``rotate``, and the primitive the anchoring transforms + are built on: ``grounded`` and ``centered`` both resolve to a call here. + For the common case of a marker sitting above the floor, prefer + ``grounded(lowest_point_height_m=...)`` over a manual shift. Args: x: Offset along the world X axis, in meters. @@ -1282,7 +1281,12 @@ def oriented(self, up: dict[int, NDArray]) -> "CaptureVolume": _optimization_status=self._optimization_status, ) - def grounded(self, mode: Literal["lowest_point"] = "lowest_point") -> "CaptureVolume": + def grounded( + self, + mode: Literal["lowest_point"] = "lowest_point", + *, + lowest_point_height_m: float = 0.0, + ) -> "CaptureVolume": """Translate so the ground sits at Z=0 and the XY origin lies under the anchor camera. ``mode="lowest_point"`` places the floor at Z=0, taken as a robust low @@ -1291,27 +1295,26 @@ def grounded(self, mode: Literal["lowest_point"] = "lowest_point") -> "CaptureVo statistic degrades to the exact minimum. Call after ``oriented()`` so Z is vertical. XY origin is set under the anchor camera (lowest posed cam_id). No rotation or scale change. + + Args: + mode: How the floor is located. Only ``"lowest_point"`` is supported. + lowest_point_height_m: How far the lowest point actually sits above + the ground, in meters. The lowest point is a marker or keypoint, + not the floor: a toe marker rides above it on the shoe and its + own thickness. Give that measured height and the floor lands at + Z=0 with the point resting above it, instead of the point being + buried at zero. Defaults to 0.0, the historical behavior. """ if mode != "lowest_point": raise ValueError(f"grounded() only supports mode='lowest_point', got {mode!r}.") min_z = float(np.percentile(self.world_points.df["z_coord"].to_numpy(), 1.0, method="lower")) anchor_center = self._camera_center(self._anchor_cam_id()) - offset = np.array([anchor_center[0], anchor_center[1], min_z], dtype=np.float64) - transform = SimilarityTransform( - rotation=np.eye(3, dtype=np.float64), - translation=-offset, - scale=1.0, - ) - new_camera_array, new_world_points = apply_similarity_transform(self.camera_array, self.world_points, transform) - - return CaptureVolume( - camera_array=new_camera_array, - image_points=self.image_points, - world_points=new_world_points, - constraints=self.constraints, - _optimization_status=self._optimization_status, + return self.translate( + x=-anchor_center[0], + y=-anchor_center[1], + z=-min_z + lowest_point_height_m, ) def centered(self) -> "CaptureVolume": @@ -1322,22 +1325,8 @@ def centered(self) -> "CaptureVolume": """ centers = np.array([self._camera_center(cid) for cid in self.camera_array.posed_cameras]) centroid_xy = centers[:, :2].mean(axis=0) - offset = np.array([centroid_xy[0], centroid_xy[1], 0.0], dtype=np.float64) - - transform = SimilarityTransform( - rotation=np.eye(3, dtype=np.float64), - translation=-offset, - scale=1.0, - ) - new_camera_array, new_world_points = apply_similarity_transform(self.camera_array, self.world_points, transform) - return CaptureVolume( - camera_array=new_camera_array, - image_points=self.image_points, - world_points=new_world_points, - constraints=self.constraints, - _optimization_status=self._optimization_status, - ) + return self.translate(x=-centroid_xy[0], y=-centroid_xy[1]) if __name__ == "__main__": diff --git a/tests/test_capture_volume_anchoring.py b/tests/test_capture_volume_anchoring.py index fda2573db..62860a9f0 100644 --- a/tests/test_capture_volume_anchoring.py +++ b/tests/test_capture_volume_anchoring.py @@ -419,6 +419,30 @@ def test_grounded_floor_matches_min_on_clean_dense_volume(): assert abs(float(grounded.world_points.df["z_coord"].min())) < 0.02 +def test_grounded_lowest_point_height_lifts_the_floor(): + """The lowest point is a marker, not the floor; its height puts the floor at zero. + + Equivalent to grounding then shifting by hand, which is what a caller had to + do before the parameter existed. + """ + cameras = { + 0: make_camera(0, np.eye(3), np.array([5.0, 7.0, 3.0])), + 1: make_camera(1, np.eye(3), np.array([8.0, 7.0, 3.0])), + } + rows = [world_row(0, 10, (1.0, 1.0, 2.0)), world_row(0, 11, (1.0, 1.0, 4.0))] + volume = make_volume(cameras, rows) + + lifted = volume.grounded(lowest_point_height_m=0.02) + + # The marker rests at its true height, so the floor it stands on is Z=0. + assert float(lifted.world_points.df["z_coord"].min()) == pytest.approx(0.02) + # XY anchoring is unaffected by the vertical correction. + assert camera_center(lifted, 0)[:2] == pytest.approx([0.0, 0.0], abs=1e-10) + + by_hand = volume.grounded().translate(z=0.02) + assert lifted.world_points.points == pytest.approx(by_hand.world_points.points) + + def test_grounded_rejects_unknown_mode(): cameras = { 0: make_camera(0, np.eye(3), np.array([0.0, 0.0, 0.0])), @@ -495,6 +519,7 @@ def test_chain_order_invariance_scale_and_orient(): debug_dir = Path(__file__).parent / "tmp" debug_dir.mkdir(parents=True, exist_ok=True) + test_grounded_lowest_point_height_lifts_the_floor() test_translate_lifts_floor_by_marker_height() test_translate_preserves_shape_and_defaults_to_no_op() test_single_camera_distance_recovers_scale() From 24e8ac070a8f7d1d4b10884d47b1994ec9844578 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 13:45:33 -0500 Subject: [PATCH 08/11] docs: rewrite the home page around what the tool does What it solves opened on pairwise PnP and transitive chaining, which is initialization detail and already covered in the extrinsic calibration page. It said nothing about intrinsics, anchoring, the GUI, the scripting API, or markerless calibration, all of which the tool now does. Replaces that with the three stages, then splits the two entry points into their own sections: with a target, and without one. --- docs/index.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/index.md b/docs/index.md index a519f1455..ec0c5f9c2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,17 +6,30 @@ BSD-2-Clause licensed. ## What it solves -Multicamera 3D reconstruction requires knowing each camera's optical properties and its position in space. -Bundle adjustment finds these, but it needs a good starting point to converge. -Caliscope builds that starting point and refines it. +Triangulating a point from several cameras means knowing each camera's optical properties and where it sits in space. +Measuring that by hand is impractical, so it has to be recovered from video of the rig itself. +Caliscope does the recovery and reports how far to trust it. -For each pair of cameras that both see the calibration target in the same frame, Caliscope estimates their relative position using PnP. -It chains these pairwise estimates transitively: if A-B and B-C are known, A-C is inferred. -The target never needs to be visible to all cameras at once. +The work runs in three stages. +Intrinsic calibration recovers focal length and lens distortion, one camera at a time. +Extrinsic calibration recovers where the cameras sit relative to each other, by bundle adjustment over the observations they share. +Anchoring then moves the solved rig into a frame you can use: vertical pointing up, distances in meters, floor at zero. -This approach supports flexible calibration targets. -A single ArUco marker on a sheet of paper can calibrate a wide capture volume. -For surround setups where cameras face inward from all directions, a charuco board printed on both sides of a rigid surface lets cameras on opposite sides link through shared points. +Everything is available two ways. +The desktop app gives visual feedback at each stage, and the [Scripting API](scripting.md) runs the same pipeline from Python. + +## Calibrating with a target + +Print a ChArUco board, an ArUco marker, or a chessboard, and record it moving through the volume. +The target never has to be visible to every camera at once, because Caliscope chains pairwise camera relationships transitively. +A single marker on a sheet of paper can calibrate a wide volume. +For surround rigs where cameras face inward from all sides, a board printed on both faces of a rigid surface links cameras that share no view. + +## Calibrating without one + +Cameras watching a moving person can be solved from the body alone, with no target. +Pose keypoints stand in for board corners, and the scene supplies the coordinate frame: an estimated vertical, one known distance for scale, and the floor. +This path needs intrinsics solved first, and it runs from the [Scripting API](scripting.md). ## Output From 8e88f4cfc9bc030186615e36448d6f491dcc68d2 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 13:49:48 -0500 Subject: [PATCH 09/11] docs: cut a filler sentence from the home page --- docs/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index ec0c5f9c2..e5c16983f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,6 @@ BSD-2-Clause licensed. Triangulating a point from several cameras means knowing each camera's optical properties and where it sits in space. Measuring that by hand is impractical, so it has to be recovered from video of the rig itself. -Caliscope does the recovery and reports how far to trust it. The work runs in three stages. Intrinsic calibration recovers focal length and lens distortion, one camera at a time. From ee397481b4a9e4e6506be6722ed9ddf21224201a Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 14:25:27 -0500 Subject: [PATCH 10/11] chore: bump version to 0.11.4 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9b7a29cb..1108a2230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "caliscope" -version = "0.11.3" +version = "0.11.4" description = "Rapid and reliable multicamera calibration with GUI and scripting API" authors = [{name = "Mac Prible", email = "prible@gmail.com"}] license = { text = "BSD-2-Clause" } diff --git a/uv.lock b/uv.lock index 7a0d0331c..b06bc452a 100644 --- a/uv.lock +++ b/uv.lock @@ -104,7 +104,7 @@ wheels = [ [[package]] name = "caliscope" -version = "0.11.3" +version = "0.11.4" source = { editable = "." } dependencies = [ { name = "av" }, From 125bda8753e343cf649404ca2fb1394d416b2f10 Mon Sep 17 00:00:00 2001 From: Mac Prible Date: Sat, 25 Jul 2026 14:26:28 -0500 Subject: [PATCH 11/11] docs: name only the constructor parameter in the printing note The full constructor takes square_size_override_cm; square_size_cm belongs to from_squares, which is the other example on the page. --- docs/scripting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripting.md b/docs/scripting.md index e88782f86..0054346a7 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -68,7 +68,7 @@ charuco.save_image("charuco.png") charuco.save_mirror_image("charuco_mirror.png") ``` -Print it, measure a square with calipers, and pass the measurement as `square_size_cm` or `square_size_override_cm`. +Print it, measure a square with calipers, and pass the measurement as `square_size_override_cm`. Printers rescale. The mirror image is the back face of a two-sided board.