From 06d576aacdfbd250d3265e3366e18ed979431cf0 Mon Sep 17 00:00:00 2001 From: Teakowa Date: Sun, 11 Oct 2026 04:23:13 +0800 Subject: [PATCH 1/3] feat(layout): read 21:9 ultrawide screenshots at the HUD's height scale Ultrawide captures were squeezed onto the 16:9 canvas, distorting every ROI. They now keep the height scale and place left, right and central ROIs at the screen edges or middle, and are no longer flagged as cropped. Rust crop-batch and inspect still use the old normalization. --- README.md | 4 +++- app/image/quality.py | 8 +++++++- app/image/roi.py | 25 +++++++++++++++++++++++-- app/service.py | 4 ++-- tests/test_image_quality.py | 8 ++++++++ tests/test_roi_selection.py | 22 ++++++++++++++++++++-- 6 files changed, 63 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d90f0f1..68520e6 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,9 @@ fields are added to the response separately from the geometry. The service selects the versioned `1280x720-v8` or `1280x800-v3` ROI layout from the source aspect ratio. The 16:10 layout is used for screenshots such as 2560x1600 without marking the -input as cropped. +input as cropped. 21:9 ultrawide screenshots (about 2.39:1, such as 3440x1440) use the +16:9 layout at the same height scale, with left, right and central ROIs kept at their screen +edges or middle, and are not marked as cropped. Send `X-Request-ID` to correlate a request; otherwise OCRKit generates one. Each field reports its parsed value, confidence, source ROI, normalization metadata, and status. diff --git a/app/image/quality.py b/app/image/quality.py index 670d4f0..b743e42 100644 --- a/app/image/quality.py +++ b/app/image/quality.py @@ -5,6 +5,8 @@ _ASPECT_TOLERANCE = 0.03 +# 3440x1440; the tolerance also covers 21:9 (2.333), 2560x1080 and 3840x1600. +ULTRAWIDE_ASPECT = 43 / 18 _BLUR_VARIANCE_REFERENCE = 1000.0 @@ -12,6 +14,10 @@ def _relative_aspect_difference(aspect_ratio: float, target_ratio: float) -> flo return abs(aspect_ratio - target_ratio) / target_ratio +def is_ultrawide(width: int, height: int) -> bool: + return _relative_aspect_difference(width / height, ULTRAWIDE_ASPECT) <= _ASPECT_TOLERANCE + + def assess_input_quality( image: np.ndarray, target_width: int, @@ -19,7 +25,7 @@ def assess_input_quality( ) -> dict[str, float | bool | str | tuple[int, int] | list[str]]: height, width = image.shape[:2] aspect_ratio = width / height - target_ratio = target_width / target_height + target_ratio = ULTRAWIDE_ASPECT if is_ultrawide(width, height) else target_width / target_height aspect_difference = _relative_aspect_difference(aspect_ratio, target_ratio) cropped = aspect_difference > _ASPECT_TOLERANCE layout_confidence = max(0.0, min(1.0, 1.0 - aspect_difference / _ASPECT_TOLERANCE)) diff --git a/app/image/roi.py b/app/image/roi.py index a7b4621..283166d 100644 --- a/app/image/roi.py +++ b/app/image/roi.py @@ -4,6 +4,7 @@ import numpy as np from app.core.roi_config import RoiBox, RoiConfig +from app.image.quality import is_ultrawide def normalize_canvas(image: np.ndarray, width: int, height: int) -> np.ndarray: @@ -22,6 +23,26 @@ def crop_roi(image: np.ndarray, box: RoiBox) -> np.ndarray: return image[box.y1:box.y2, box.x1:box.x2].copy() +def anchored_rois(config: RoiConfig, canvas_width: int) -> dict[str, RoiBox]: + """Place ROIs on a canvas wider than the standard one. + + The HUD keeps its height scale and stays attached to the screen edges, so + left ROIs stay put, right ROIs follow the right edge and central ROIs the middle. + """ + extra = canvas_width - config.width + shifted: dict[str, RoiBox] = {} + for name, box in config.rois.items(): + centre = (box.x1 + box.x2) / 2 + shift = 0 if centre < config.width / 3 else extra if centre > config.width * 2 / 3 else extra // 2 + shifted[name] = RoiBox(box.x1 + shift, box.y1, box.x2 + shift, box.y2) + return shifted + + def crop_all_rois(image: np.ndarray, config: RoiConfig) -> tuple[np.ndarray, dict[str, np.ndarray]]: - normalized = normalize_canvas(image, config.width, config.height) - return normalized, {name: crop_roi(normalized, box) for name, box in config.rois.items()} + height, width = image.shape[:2] + if is_ultrawide(width, height): + normalized = normalize_canvas(image, round(config.height * width / height), config.height) + else: + normalized = normalize_canvas(image, config.width, config.height) + rois = anchored_rois(config, normalized.shape[1]) + return normalized, {name: crop_roi(normalized, box) for name, box in rois.items()} diff --git a/app/service.py b/app/service.py index ab73bbd..d1fd9bf 100644 --- a/app/service.py +++ b/app/service.py @@ -7,7 +7,7 @@ from app.core.roi_config import RoiConfig from app.image.preprocess import preprocess_by_roi from app.image.quality import assess_input_quality -from app.image.roi import crop_all_rois, select_roi_config +from app.image.roi import anchored_rois, crop_all_rois, select_roi_config from app.ocr.engine import OcrChunk, OcrEngine from app.parser.ai_mark import is_ai_mark from app.parser.bottom_left_hero import parse_bottom_left_hero @@ -211,7 +211,7 @@ def extract_structured( "x2": box.x2, "y2": box.y2, } - for name, box in active_roi_config.rois.items() + for name, box in anchored_rois(active_roi_config, normalized.shape[1]).items() }, raw_text=raw_text, confidence=confidences, diff --git a/tests/test_image_quality.py b/tests/test_image_quality.py index 5811aae..90485f9 100644 --- a/tests/test_image_quality.py +++ b/tests/test_image_quality.py @@ -16,6 +16,14 @@ def test_quality_preserves_original_size_and_reports_supported_layout() -> None: assert quality["warnings"] == [] +def test_quality_accepts_ultrawide_but_still_flags_other_wide_crops() -> None: + for width, height in ((3440, 1440), (2560, 1080), (3840, 1600)): + quality = assess_input_quality(np.zeros((height, width, 3), dtype=np.uint8), 1280, 720) + assert quality["cropped"] is False, (width, height) + assert quality["warnings"] == [] + assert assess_input_quality(np.zeros((1000, 2000, 3), dtype=np.uint8), 1280, 720)["cropped"] is True + + def test_quality_reports_aspect_ratio_risk_before_normalization() -> None: image = np.zeros((1000, 1500, 3), dtype=np.uint8) diff --git a/tests/test_roi_selection.py b/tests/test_roi_selection.py index 312913a..6a21686 100644 --- a/tests/test_roi_selection.py +++ b/tests/test_roi_selection.py @@ -1,7 +1,9 @@ +from pathlib import Path + import numpy as np -from app.core.roi_config import RoiBox, RoiConfig -from app.image.roi import select_roi_config +from app.core.roi_config import RoiBox, RoiConfig, load_roi_config +from app.image.roi import anchored_rois, crop_all_rois, select_roi_config def _config(width: int, height: int, version: str) -> RoiConfig: @@ -14,3 +16,19 @@ def test_select_roi_config_uses_matching_source_aspect_ratio() -> None: assert select_roi_config(np.zeros((1600, 2560, 3), dtype=np.uint8), (widescreen, tall)) == tall assert select_roi_config(np.zeros((1440, 2560, 3), dtype=np.uint8), (widescreen, tall)) == widescreen + + +def test_ultrawide_keeps_height_scale_and_anchors_hud_to_screen_edges() -> None: + config = load_roi_config(Path("configs/roi_1280x720.yaml")) + rng = np.random.default_rng(7) + standard = rng.integers(0, 256, (1440, 2560, 3), dtype=np.uint8) + extra = rng.integers(0, 256, (1440, 880, 3), dtype=np.uint8) + ultrawide = np.concatenate([standard[:, :1280], extra, standard[:, 1280:]], axis=1) + + _, standard_crops = crop_all_rois(standard, config) + normalized, wide_crops = crop_all_rois(ultrawide, config) + + assert normalized.shape[:2] == (720, 1720) + for name in ("left_panel", "run_code_panel", "achievement_panel", "bottom_left_hero", "right_panel", "run_code_right_panel"): + assert np.array_equal(wide_crops[name], standard_crops[name]), name + assert anchored_rois(config, 1720)["right_panel"].x2 == config.rois["right_panel"].x2 + 440 From b073f02dec81cdf24336cdd55d332601cb8f821f Mon Sep 17 00:00:00 2001 From: Teakowa Date: Sun, 11 Oct 2026 04:26:14 +0800 Subject: [PATCH 2/3] feat(rust): align crop-batch and inspect with the ultrawide layout Same canvas, anchoring and quality rules as the Python service, so Studio crops 21:9 screenshots where recognition reads them. --- app/image/roi.py | 2 +- rust/README.md | 4 +- rust/ocrkit-image-cli/src/main.rs | 16 ++--- rust/ocrkit-image-core/src/lib.rs | 116 +++++++++++++++++++++++++++--- 4 files changed, 117 insertions(+), 21 deletions(-) diff --git a/app/image/roi.py b/app/image/roi.py index 283166d..5c138c4 100644 --- a/app/image/roi.py +++ b/app/image/roi.py @@ -41,7 +41,7 @@ def anchored_rois(config: RoiConfig, canvas_width: int) -> dict[str, RoiBox]: def crop_all_rois(image: np.ndarray, config: RoiConfig) -> tuple[np.ndarray, dict[str, np.ndarray]]: height, width = image.shape[:2] if is_ultrawide(width, height): - normalized = normalize_canvas(image, round(config.height * width / height), config.height) + normalized = normalize_canvas(image, (config.height * width * 2 + height) // (2 * height), config.height) else: normalized = normalize_canvas(image, config.width, config.height) rois = anchored_rois(config, normalized.shape[1]) diff --git a/rust/README.md b/rust/README.md index 57bf2da..3b6f17a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -22,7 +22,9 @@ uv run python scripts/export_layout_manifest.py \ ~~~ The native preflight CLI validates the manifest, evaluates the source aspect -ratio, and maps standard-layout ROI coordinates onto the source image: +ratio, and maps standard-layout ROI coordinates onto the source image. 21:9 ultrawide +sources keep the standard height scale and place left, right and central ROIs at the screen +edges or middle, matching the Python service: ~~~bash cargo run --manifest-path rust/Cargo.toml -p ocrkit-image-cli -- \ diff --git a/rust/ocrkit-image-cli/src/main.rs b/rust/ocrkit-image-cli/src/main.rs index 3638f4d..5079eeb 100644 --- a/rust/ocrkit-image-cli/src/main.rs +++ b/rust/ocrkit-image-cli/src/main.rs @@ -1,5 +1,7 @@ use image::{imageops::FilterType, DynamicImage, GenericImageView, ImageFormat, ImageReader}; -use ocrkit_image_core::{assess_quality, ImageSize, LayoutManifest, QualityReport, RoiBox}; +use ocrkit_image_core::{ + assess_quality, canvas_size, ImageSize, LayoutManifest, QualityReport, RoiBox, +}; use serde::Deserialize; use sha2::{Digest, Sha256}; use std::env; @@ -125,14 +127,12 @@ fn crop_batch(args: &[String]) -> Result { let source_size = image_size(&source_image); let quality = assess_quality(source_size, manifest.standard_size).map_err(|err| err.to_string())?; - let normalized = source_image.resize_exact( - manifest.standard_size.width, - manifest.standard_size.height, - FilterType::Triangle, - ); + let canvas = canvas_size(source_size, manifest.standard_size); + let normalized = + source_image.resize_exact(canvas.width, canvas.height, FilterType::Triangle); let mut rois = std::collections::BTreeMap::new(); - for (roi_name, roi) in &manifest.rois { + for (roi_name, roi) in &manifest.anchored_rois(canvas) { validate_component(roi_name, "ROI name")?; let crop = normalized.crop_imm(roi.x1, roi.y1, roi.width(), roi.height()); let relative_path = PathBuf::from("images") @@ -164,7 +164,7 @@ fn crop_batch(args: &[String]) -> Result { source_file: case.image, source_sha256, source_size, - normalized_size: manifest.standard_size, + normalized_size: canvas, quality, rois, }); diff --git a/rust/ocrkit-image-core/src/lib.rs b/rust/ocrkit-image-core/src/lib.rs index 4230d61..f4a8dc2 100644 --- a/rust/ocrkit-image-core/src/lib.rs +++ b/rust/ocrkit-image-core/src/lib.rs @@ -18,6 +18,30 @@ impl ImageSize { } } +/// 3440x1440; the 3% tolerance also covers 21:9, 2560x1080 and 3840x1600. +const ULTRAWIDE_ASPECT: f64 = 43.0 / 18.0; +const ASPECT_TOLERANCE: f64 = 0.03; + +fn relative_aspect_difference(aspect_ratio: f64, target_ratio: f64) -> f64 { + (aspect_ratio - target_ratio).abs() / target_ratio +} + +pub fn is_ultrawide(source: ImageSize) -> bool { + source.height > 0 + && relative_aspect_difference(source.width as f64 / source.height as f64, ULTRAWIDE_ASPECT) + <= ASPECT_TOLERANCE +} + +/// The canvas a source is normalized onto: ultrawide sources keep the standard height scale. +pub fn canvas_size(source: ImageSize, standard: ImageSize) -> ImageSize { + if !is_ultrawide(source) { + return standard; + } + let numerator = standard.height as u64 * source.width as u64; + let width = (numerator * 2 + source.height as u64) / (2 * source.height as u64); + ImageSize::new(width as u32, standard.height) +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct RoiBox { pub x1: u32, @@ -39,6 +63,21 @@ impl RoiBox { self.y2.saturating_sub(self.y1) } + /// Place the ROI on a canvas wider than the standard one: left ROIs stay put, right ROIs + /// follow the right edge and central ROIs the middle. + pub fn anchored(self, standard: ImageSize, canvas_width: u32) -> Self { + let extra = canvas_width.saturating_sub(standard.width); + let doubled_centre = (self.x1 + self.x2) as u64 * 3; + let shift = if doubled_centre < 2 * standard.width as u64 { + 0 + } else if doubled_centre > 4 * standard.width as u64 { + extra + } else { + extra / 2 + }; + Self::new(self.x1 + shift, self.y1, self.x2 + shift, self.y2) + } + pub fn scaled_to(self, source: ImageSize, standard: ImageSize) -> Result { if source.width == 0 || source.height == 0 { return Err(LayoutError::InvalidImageSize(source)); @@ -94,17 +133,22 @@ impl LayoutManifest { Ok(()) } + pub fn anchored_rois(&self, canvas: ImageSize) -> BTreeMap { + self.rois + .iter() + .map(|(name, roi)| (name.clone(), roi.anchored(self.standard_size, canvas.width))) + .collect() + } + pub fn scaled_rois(&self, source: ImageSize) -> Result, LayoutError> { self.validate()?; if source.width == 0 || source.height == 0 { return Err(LayoutError::InvalidImageSize(source)); } - self.rois - .iter() - .map(|(name, roi)| { - roi.scaled_to(source, self.standard_size) - .map(|scaled| (name.clone(), scaled)) - }) + let canvas = canvas_size(source, self.standard_size); + self.anchored_rois(canvas) + .into_iter() + .map(|(name, roi)| roi.scaled_to(source, canvas).map(|scaled| (name, scaled))) .collect() } } @@ -127,11 +171,14 @@ pub fn assess_quality(source: ImageSize, target: ImageSize) -> Result tolerance; - let layout_confidence = (1.0 - aspect_difference / tolerance).clamp(0.0, 1.0); + let target_ratio = if is_ultrawide(source) { + ULTRAWIDE_ASPECT + } else { + target.width as f64 / target.height as f64 + }; + let aspect_difference = relative_aspect_difference(aspect_ratio, target_ratio); + let cropped = aspect_difference > ASPECT_TOLERANCE; + let layout_confidence = (1.0 - aspect_difference / ASPECT_TOLERANCE).clamp(0.0, 1.0); let warnings = if cropped { vec![ "quality.aspect_ratio_mismatch".to_string(), @@ -234,6 +281,53 @@ mod tests { )); } + #[test] + fn ultrawide_keeps_height_scale_and_anchors_rois_to_screen_edges() { + let mut layout = manifest(); + layout + .rois + .insert("left_panel".to_string(), RoiBox::new(35, 15, 220, 225)); + layout.rois.insert( + "center_banner".to_string(), + RoiBox::new(150, 125, 1160, 215), + ); + let source = ImageSize::new(3440, 1440); + + assert_eq!( + canvas_size(source, layout.standard_size), + ImageSize::new(1720, 720) + ); + assert_eq!( + canvas_size(ImageSize::new(2560, 1440), layout.standard_size), + layout.standard_size + ); + let anchored = layout.anchored_rois(ImageSize::new(1720, 720)); + assert_eq!(anchored["left_panel"], RoiBox::new(35, 15, 220, 225)); + assert_eq!(anchored["center_banner"], RoiBox::new(370, 125, 1380, 215)); + assert_eq!(anchored["right_panel"], RoiBox::new(1480, 0, 1715, 105)); + assert_eq!( + layout.scaled_rois(source).unwrap()["right_panel"], + RoiBox::new(2960, 0, 3430, 210) + ); + } + + #[test] + fn ultrawide_is_not_flagged_as_cropped() { + for (width, height) in [(3440, 1440), (2560, 1080), (3840, 1600)] { + let quality = + assess_quality(ImageSize::new(width, height), ImageSize::new(1280, 720)).unwrap(); + assert!( + !quality.cropped && quality.warnings.is_empty(), + "{width}x{height}" + ); + } + assert!( + assess_quality(ImageSize::new(2000, 1000), ImageSize::new(1280, 720)) + .unwrap() + .cropped + ); + } + #[test] fn quality_matches_python_aspect_contract() { let quality = From 65ab5e86a47ca879aa5795c9370e8036eafc430b Mon Sep 17 00:00:00 2001 From: Teakowa Date: Sun, 11 Oct 2026 04:42:30 +0800 Subject: [PATCH 3/3] test(layout): cover the AI watermark and server load ROIs in the ultrawide anchoring check --- tests/test_roi_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_roi_selection.py b/tests/test_roi_selection.py index 6a21686..1cb6b9e 100644 --- a/tests/test_roi_selection.py +++ b/tests/test_roi_selection.py @@ -29,6 +29,6 @@ def test_ultrawide_keeps_height_scale_and_anchors_hud_to_screen_edges() -> None: normalized, wide_crops = crop_all_rois(ultrawide, config) assert normalized.shape[:2] == (720, 1720) - for name in ("left_panel", "run_code_panel", "achievement_panel", "bottom_left_hero", "right_panel", "run_code_right_panel"): + for name in ("left_panel", "run_code_panel", "achievement_panel", "bottom_left_hero", "right_panel", "run_code_right_panel", "ai_watermark", "server_load_panel"): assert np.array_equal(wide_crops[name], standard_crops[name]), name assert anchored_rois(config, 1720)["right_panel"].x2 == config.rois["right_panel"].x2 + 440