diff --git a/README.md b/README.md index d90f0f1..68520e6 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,9 @@ fields are added to the response separately from the geometry. The service selects the versioned `1280x720-v8` or `1280x800-v3` ROI layout from the source aspect ratio. The 16:10 layout is used for screenshots such as 2560x1600 without marking the -input as cropped. +input as cropped. 21:9 ultrawide screenshots (about 2.39:1, such as 3440x1440) use the +16:9 layout at the same height scale, with left, right and central ROIs kept at their screen +edges or middle, and are not marked as cropped. Send `X-Request-ID` to correlate a request; otherwise OCRKit generates one. Each field reports its parsed value, confidence, source ROI, normalization metadata, and status. diff --git a/app/image/quality.py b/app/image/quality.py index 670d4f0..b743e42 100644 --- a/app/image/quality.py +++ b/app/image/quality.py @@ -5,6 +5,8 @@ _ASPECT_TOLERANCE = 0.03 +# 3440x1440; the tolerance also covers 21:9 (2.333), 2560x1080 and 3840x1600. +ULTRAWIDE_ASPECT = 43 / 18 _BLUR_VARIANCE_REFERENCE = 1000.0 @@ -12,6 +14,10 @@ def _relative_aspect_difference(aspect_ratio: float, target_ratio: float) -> flo return abs(aspect_ratio - target_ratio) / target_ratio +def is_ultrawide(width: int, height: int) -> bool: + return _relative_aspect_difference(width / height, ULTRAWIDE_ASPECT) <= _ASPECT_TOLERANCE + + def assess_input_quality( image: np.ndarray, target_width: int, @@ -19,7 +25,7 @@ def assess_input_quality( ) -> dict[str, float | bool | str | tuple[int, int] | list[str]]: height, width = image.shape[:2] aspect_ratio = width / height - target_ratio = target_width / target_height + target_ratio = ULTRAWIDE_ASPECT if is_ultrawide(width, height) else target_width / target_height aspect_difference = _relative_aspect_difference(aspect_ratio, target_ratio) cropped = aspect_difference > _ASPECT_TOLERANCE layout_confidence = max(0.0, min(1.0, 1.0 - aspect_difference / _ASPECT_TOLERANCE)) diff --git a/app/image/roi.py b/app/image/roi.py index a7b4621..5c138c4 100644 --- a/app/image/roi.py +++ b/app/image/roi.py @@ -4,6 +4,7 @@ import numpy as np from app.core.roi_config import RoiBox, RoiConfig +from app.image.quality import is_ultrawide def normalize_canvas(image: np.ndarray, width: int, height: int) -> np.ndarray: @@ -22,6 +23,26 @@ def crop_roi(image: np.ndarray, box: RoiBox) -> np.ndarray: return image[box.y1:box.y2, box.x1:box.x2].copy() +def anchored_rois(config: RoiConfig, canvas_width: int) -> dict[str, RoiBox]: + """Place ROIs on a canvas wider than the standard one. + + The HUD keeps its height scale and stays attached to the screen edges, so + left ROIs stay put, right ROIs follow the right edge and central ROIs the middle. + """ + extra = canvas_width - config.width + shifted: dict[str, RoiBox] = {} + for name, box in config.rois.items(): + centre = (box.x1 + box.x2) / 2 + shift = 0 if centre < config.width / 3 else extra if centre > config.width * 2 / 3 else extra // 2 + shifted[name] = RoiBox(box.x1 + shift, box.y1, box.x2 + shift, box.y2) + return shifted + + def crop_all_rois(image: np.ndarray, config: RoiConfig) -> tuple[np.ndarray, dict[str, np.ndarray]]: - normalized = normalize_canvas(image, config.width, config.height) - return normalized, {name: crop_roi(normalized, box) for name, box in config.rois.items()} + height, width = image.shape[:2] + if is_ultrawide(width, height): + normalized = normalize_canvas(image, (config.height * width * 2 + height) // (2 * height), config.height) + else: + normalized = normalize_canvas(image, config.width, config.height) + rois = anchored_rois(config, normalized.shape[1]) + return normalized, {name: crop_roi(normalized, box) for name, box in rois.items()} diff --git a/app/service.py b/app/service.py index ab73bbd..d1fd9bf 100644 --- a/app/service.py +++ b/app/service.py @@ -7,7 +7,7 @@ from app.core.roi_config import RoiConfig from app.image.preprocess import preprocess_by_roi from app.image.quality import assess_input_quality -from app.image.roi import crop_all_rois, select_roi_config +from app.image.roi import anchored_rois, crop_all_rois, select_roi_config from app.ocr.engine import OcrChunk, OcrEngine from app.parser.ai_mark import is_ai_mark from app.parser.bottom_left_hero import parse_bottom_left_hero @@ -211,7 +211,7 @@ def extract_structured( "x2": box.x2, "y2": box.y2, } - for name, box in active_roi_config.rois.items() + for name, box in anchored_rois(active_roi_config, normalized.shape[1]).items() }, raw_text=raw_text, confidence=confidences, diff --git a/rust/README.md b/rust/README.md index 57bf2da..3b6f17a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -22,7 +22,9 @@ uv run python scripts/export_layout_manifest.py \ ~~~ The native preflight CLI validates the manifest, evaluates the source aspect -ratio, and maps standard-layout ROI coordinates onto the source image: +ratio, and maps standard-layout ROI coordinates onto the source image. 21:9 ultrawide +sources keep the standard height scale and place left, right and central ROIs at the screen +edges or middle, matching the Python service: ~~~bash cargo run --manifest-path rust/Cargo.toml -p ocrkit-image-cli -- \ diff --git a/rust/ocrkit-image-cli/src/main.rs b/rust/ocrkit-image-cli/src/main.rs index 3638f4d..5079eeb 100644 --- a/rust/ocrkit-image-cli/src/main.rs +++ b/rust/ocrkit-image-cli/src/main.rs @@ -1,5 +1,7 @@ use image::{imageops::FilterType, DynamicImage, GenericImageView, ImageFormat, ImageReader}; -use ocrkit_image_core::{assess_quality, ImageSize, LayoutManifest, QualityReport, RoiBox}; +use ocrkit_image_core::{ + assess_quality, canvas_size, ImageSize, LayoutManifest, QualityReport, RoiBox, +}; use serde::Deserialize; use sha2::{Digest, Sha256}; use std::env; @@ -125,14 +127,12 @@ fn crop_batch(args: &[String]) -> Result { let source_size = image_size(&source_image); let quality = assess_quality(source_size, manifest.standard_size).map_err(|err| err.to_string())?; - let normalized = source_image.resize_exact( - manifest.standard_size.width, - manifest.standard_size.height, - FilterType::Triangle, - ); + let canvas = canvas_size(source_size, manifest.standard_size); + let normalized = + source_image.resize_exact(canvas.width, canvas.height, FilterType::Triangle); let mut rois = std::collections::BTreeMap::new(); - for (roi_name, roi) in &manifest.rois { + for (roi_name, roi) in &manifest.anchored_rois(canvas) { validate_component(roi_name, "ROI name")?; let crop = normalized.crop_imm(roi.x1, roi.y1, roi.width(), roi.height()); let relative_path = PathBuf::from("images") @@ -164,7 +164,7 @@ fn crop_batch(args: &[String]) -> Result { source_file: case.image, source_sha256, source_size, - normalized_size: manifest.standard_size, + normalized_size: canvas, quality, rois, }); diff --git a/rust/ocrkit-image-core/src/lib.rs b/rust/ocrkit-image-core/src/lib.rs index 4230d61..f4a8dc2 100644 --- a/rust/ocrkit-image-core/src/lib.rs +++ b/rust/ocrkit-image-core/src/lib.rs @@ -18,6 +18,30 @@ impl ImageSize { } } +/// 3440x1440; the 3% tolerance also covers 21:9, 2560x1080 and 3840x1600. +const ULTRAWIDE_ASPECT: f64 = 43.0 / 18.0; +const ASPECT_TOLERANCE: f64 = 0.03; + +fn relative_aspect_difference(aspect_ratio: f64, target_ratio: f64) -> f64 { + (aspect_ratio - target_ratio).abs() / target_ratio +} + +pub fn is_ultrawide(source: ImageSize) -> bool { + source.height > 0 + && relative_aspect_difference(source.width as f64 / source.height as f64, ULTRAWIDE_ASPECT) + <= ASPECT_TOLERANCE +} + +/// The canvas a source is normalized onto: ultrawide sources keep the standard height scale. +pub fn canvas_size(source: ImageSize, standard: ImageSize) -> ImageSize { + if !is_ultrawide(source) { + return standard; + } + let numerator = standard.height as u64 * source.width as u64; + let width = (numerator * 2 + source.height as u64) / (2 * source.height as u64); + ImageSize::new(width as u32, standard.height) +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct RoiBox { pub x1: u32, @@ -39,6 +63,21 @@ impl RoiBox { self.y2.saturating_sub(self.y1) } + /// Place the ROI on a canvas wider than the standard one: left ROIs stay put, right ROIs + /// follow the right edge and central ROIs the middle. + pub fn anchored(self, standard: ImageSize, canvas_width: u32) -> Self { + let extra = canvas_width.saturating_sub(standard.width); + let doubled_centre = (self.x1 + self.x2) as u64 * 3; + let shift = if doubled_centre < 2 * standard.width as u64 { + 0 + } else if doubled_centre > 4 * standard.width as u64 { + extra + } else { + extra / 2 + }; + Self::new(self.x1 + shift, self.y1, self.x2 + shift, self.y2) + } + pub fn scaled_to(self, source: ImageSize, standard: ImageSize) -> Result { if source.width == 0 || source.height == 0 { return Err(LayoutError::InvalidImageSize(source)); @@ -94,17 +133,22 @@ impl LayoutManifest { Ok(()) } + pub fn anchored_rois(&self, canvas: ImageSize) -> BTreeMap { + self.rois + .iter() + .map(|(name, roi)| (name.clone(), roi.anchored(self.standard_size, canvas.width))) + .collect() + } + pub fn scaled_rois(&self, source: ImageSize) -> Result, LayoutError> { self.validate()?; if source.width == 0 || source.height == 0 { return Err(LayoutError::InvalidImageSize(source)); } - self.rois - .iter() - .map(|(name, roi)| { - roi.scaled_to(source, self.standard_size) - .map(|scaled| (name.clone(), scaled)) - }) + let canvas = canvas_size(source, self.standard_size); + self.anchored_rois(canvas) + .into_iter() + .map(|(name, roi)| roi.scaled_to(source, canvas).map(|scaled| (name, scaled))) .collect() } } @@ -127,11 +171,14 @@ pub fn assess_quality(source: ImageSize, target: ImageSize) -> Result tolerance; - let layout_confidence = (1.0 - aspect_difference / tolerance).clamp(0.0, 1.0); + let target_ratio = if is_ultrawide(source) { + ULTRAWIDE_ASPECT + } else { + target.width as f64 / target.height as f64 + }; + let aspect_difference = relative_aspect_difference(aspect_ratio, target_ratio); + let cropped = aspect_difference > ASPECT_TOLERANCE; + let layout_confidence = (1.0 - aspect_difference / ASPECT_TOLERANCE).clamp(0.0, 1.0); let warnings = if cropped { vec![ "quality.aspect_ratio_mismatch".to_string(), @@ -234,6 +281,53 @@ mod tests { )); } + #[test] + fn ultrawide_keeps_height_scale_and_anchors_rois_to_screen_edges() { + let mut layout = manifest(); + layout + .rois + .insert("left_panel".to_string(), RoiBox::new(35, 15, 220, 225)); + layout.rois.insert( + "center_banner".to_string(), + RoiBox::new(150, 125, 1160, 215), + ); + let source = ImageSize::new(3440, 1440); + + assert_eq!( + canvas_size(source, layout.standard_size), + ImageSize::new(1720, 720) + ); + assert_eq!( + canvas_size(ImageSize::new(2560, 1440), layout.standard_size), + layout.standard_size + ); + let anchored = layout.anchored_rois(ImageSize::new(1720, 720)); + assert_eq!(anchored["left_panel"], RoiBox::new(35, 15, 220, 225)); + assert_eq!(anchored["center_banner"], RoiBox::new(370, 125, 1380, 215)); + assert_eq!(anchored["right_panel"], RoiBox::new(1480, 0, 1715, 105)); + assert_eq!( + layout.scaled_rois(source).unwrap()["right_panel"], + RoiBox::new(2960, 0, 3430, 210) + ); + } + + #[test] + fn ultrawide_is_not_flagged_as_cropped() { + for (width, height) in [(3440, 1440), (2560, 1080), (3840, 1600)] { + let quality = + assess_quality(ImageSize::new(width, height), ImageSize::new(1280, 720)).unwrap(); + assert!( + !quality.cropped && quality.warnings.is_empty(), + "{width}x{height}" + ); + } + assert!( + assess_quality(ImageSize::new(2000, 1000), ImageSize::new(1280, 720)) + .unwrap() + .cropped + ); + } + #[test] fn quality_matches_python_aspect_contract() { let quality = diff --git a/tests/test_image_quality.py b/tests/test_image_quality.py index 5811aae..90485f9 100644 --- a/tests/test_image_quality.py +++ b/tests/test_image_quality.py @@ -16,6 +16,14 @@ def test_quality_preserves_original_size_and_reports_supported_layout() -> None: assert quality["warnings"] == [] +def test_quality_accepts_ultrawide_but_still_flags_other_wide_crops() -> None: + for width, height in ((3440, 1440), (2560, 1080), (3840, 1600)): + quality = assess_input_quality(np.zeros((height, width, 3), dtype=np.uint8), 1280, 720) + assert quality["cropped"] is False, (width, height) + assert quality["warnings"] == [] + assert assess_input_quality(np.zeros((1000, 2000, 3), dtype=np.uint8), 1280, 720)["cropped"] is True + + def test_quality_reports_aspect_ratio_risk_before_normalization() -> None: image = np.zeros((1000, 1500, 3), dtype=np.uint8) diff --git a/tests/test_roi_selection.py b/tests/test_roi_selection.py index 312913a..1cb6b9e 100644 --- a/tests/test_roi_selection.py +++ b/tests/test_roi_selection.py @@ -1,7 +1,9 @@ +from pathlib import Path + import numpy as np -from app.core.roi_config import RoiBox, RoiConfig -from app.image.roi import select_roi_config +from app.core.roi_config import RoiBox, RoiConfig, load_roi_config +from app.image.roi import anchored_rois, crop_all_rois, select_roi_config def _config(width: int, height: int, version: str) -> RoiConfig: @@ -14,3 +16,19 @@ def test_select_roi_config_uses_matching_source_aspect_ratio() -> None: assert select_roi_config(np.zeros((1600, 2560, 3), dtype=np.uint8), (widescreen, tall)) == tall assert select_roi_config(np.zeros((1440, 2560, 3), dtype=np.uint8), (widescreen, tall)) == widescreen + + +def test_ultrawide_keeps_height_scale_and_anchors_hud_to_screen_edges() -> None: + config = load_roi_config(Path("configs/roi_1280x720.yaml")) + rng = np.random.default_rng(7) + standard = rng.integers(0, 256, (1440, 2560, 3), dtype=np.uint8) + extra = rng.integers(0, 256, (1440, 880, 3), dtype=np.uint8) + ultrawide = np.concatenate([standard[:, :1280], extra, standard[:, 1280:]], axis=1) + + _, standard_crops = crop_all_rois(standard, config) + normalized, wide_crops = crop_all_rois(ultrawide, config) + + assert normalized.shape[:2] == (720, 1720) + for name in ("left_panel", "run_code_panel", "achievement_panel", "bottom_left_hero", "right_panel", "run_code_right_panel", "ai_watermark", "server_load_panel"): + assert np.array_equal(wide_crops[name], standard_crops[name]), name + assert anchored_rois(config, 1720)["right_panel"].x2 == config.rois["right_panel"].x2 + 440