feat(layout): read 21:9 ultrawide screenshots at the HUD's height scale

README.md

@@ -128,7 +128,9 @@ fields are added to the response separately from the geometry.
 
 The service selects the versioned `1280x720-v8` or `1280x800-v3` ROI layout from the source
 aspect ratio. The 16:10 layout is used for screenshots such as 2560x1600 without marking the
-input as cropped.
+input as cropped. 21:9 ultrawide screenshots (about 2.39:1, such as 3440x1440) use the
+16:9 layout at the same height scale, with left, right and central ROIs kept at their screen
+edges or middle, and are not marked as cropped.
 
 Send `X-Request-ID` to correlate a request; otherwise OCRKit generates one. Each field
 reports its parsed value, confidence, source ROI, normalization metadata, and status.

app/image/quality.py

@@ -5,21 +5,27 @@
 
 
 _ASPECT_TOLERANCE = 0.03
+# 3440x1440; the tolerance also covers 21:9 (2.333), 2560x1080 and 3840x1600.
+ULTRAWIDE_ASPECT = 43 / 18
 _BLUR_VARIANCE_REFERENCE = 1000.0
 
 
 def _relative_aspect_difference(aspect_ratio: float, target_ratio: float) -> float:
     return abs(aspect_ratio - target_ratio) / target_ratio
 
 
+def is_ultrawide(width: int, height: int) -> bool:
+    return _relative_aspect_difference(width / height, ULTRAWIDE_ASPECT) <= _ASPECT_TOLERANCE
+
+
 def assess_input_quality(
     image: np.ndarray,
     target_width: int,
     target_height: int,
 ) -> dict[str, float | bool | str | tuple[int, int] | list[str]]:
     height, width = image.shape[:2]
     aspect_ratio = width / height
-    target_ratio = target_width / target_height
+    target_ratio = ULTRAWIDE_ASPECT if is_ultrawide(width, height) else target_width / target_height
     aspect_difference = _relative_aspect_difference(aspect_ratio, target_ratio)
     cropped = aspect_difference > _ASPECT_TOLERANCE
     layout_confidence = max(0.0, min(1.0, 1.0 - aspect_difference / _ASPECT_TOLERANCE))

app/image/roi.py

@@ -4,6 +4,7 @@
 import numpy as np
 
 from app.core.roi_config import RoiBox, RoiConfig
+from app.image.quality import is_ultrawide
 
 
 def normalize_canvas(image: np.ndarray, width: int, height: int) -> np.ndarray:
@@ -22,6 +23,26 @@ def crop_roi(image: np.ndarray, box: RoiBox) -> np.ndarray:
     return image[box.y1:box.y2, box.x1:box.x2].copy()
 
 
+def anchored_rois(config: RoiConfig, canvas_width: int) -> dict[str, RoiBox]:
+    """Place ROIs on a canvas wider than the standard one.
+
+    The HUD keeps its height scale and stays attached to the screen edges, so
+    left ROIs stay put, right ROIs follow the right edge and central ROIs the middle.
+    """
+    extra = canvas_width - config.width
+    shifted: dict[str, RoiBox] = {}
+    for name, box in config.rois.items():
+        centre = (box.x1 + box.x2) / 2
+        shift = 0 if centre < config.width / 3 else extra if centre > config.width * 2 / 3 else extra // 2
+        shifted[name] = RoiBox(box.x1 + shift, box.y1, box.x2 + shift, box.y2)
+    return shifted
+
+
 def crop_all_rois(image: np.ndarray, config: RoiConfig) -> tuple[np.ndarray, dict[str, np.ndarray]]:
-    normalized = normalize_canvas(image, config.width, config.height)
-    return normalized, {name: crop_roi(normalized, box) for name, box in config.rois.items()}
+    height, width = image.shape[:2]
+    if is_ultrawide(width, height):
+        normalized = normalize_canvas(image, (config.height * width * 2 + height) // (2 * height), config.height)
+    else:
+        normalized = normalize_canvas(image, config.width, config.height)
+    rois = anchored_rois(config, normalized.shape[1])
+    return normalized, {name: crop_roi(normalized, box) for name, box in rois.items()}

app/service.py

@@ -7,7 +7,7 @@
 from app.core.roi_config import RoiConfig
 from app.image.preprocess import preprocess_by_roi
 from app.image.quality import assess_input_quality
-from app.image.roi import crop_all_rois, select_roi_config
+from app.image.roi import anchored_rois, crop_all_rois, select_roi_config
 from app.ocr.engine import OcrChunk, OcrEngine
 from app.parser.ai_mark import is_ai_mark
 from app.parser.bottom_left_hero import parse_bottom_left_hero
@@ -211,7 +211,7 @@ def extract_structured(
                     "x2": box.x2,
                     "y2": box.y2,
                 }
-                for name, box in active_roi_config.rois.items()
+                for name, box in anchored_rois(active_roi_config, normalized.shape[1]).items()
             },
             raw_text=raw_text,
             confidence=confidences,

rust/README.md

@@ -22,7 +22,9 @@ uv run python scripts/export_layout_manifest.py \
 ~~~
 
 The native preflight CLI validates the manifest, evaluates the source aspect
-ratio, and maps standard-layout ROI coordinates onto the source image:
+ratio, and maps standard-layout ROI coordinates onto the source image. 21:9 ultrawide
+sources keep the standard height scale and place left, right and central ROIs at the screen
+edges or middle, matching the Python service:
 
 ~~~bash
 cargo run --manifest-path rust/Cargo.toml -p ocrkit-image-cli -- \

rust/ocrkit-image-cli/src/main.rs

@@ -1,5 +1,7 @@
 use image::{imageops::FilterType, DynamicImage, GenericImageView, ImageFormat, ImageReader};
-use ocrkit_image_core::{assess_quality, ImageSize, LayoutManifest, QualityReport, RoiBox};
+use ocrkit_image_core::{
+    assess_quality, canvas_size, ImageSize, LayoutManifest, QualityReport, RoiBox,
+};
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
 use std::env;
@@ -125,14 +127,12 @@ fn crop_batch(args: &[String]) -> Result<String, String> {
         let source_size = image_size(&source_image);
         let quality =
             assess_quality(source_size, manifest.standard_size).map_err(|err| err.to_string())?;
-        let normalized = source_image.resize_exact(
-            manifest.standard_size.width,
-            manifest.standard_size.height,
-            FilterType::Triangle,
-        );
+        let canvas = canvas_size(source_size, manifest.standard_size);
+        let normalized =
+            source_image.resize_exact(canvas.width, canvas.height, FilterType::Triangle);
 
         let mut rois = std::collections::BTreeMap::new();
-        for (roi_name, roi) in &manifest.rois {
+        for (roi_name, roi) in &manifest.anchored_rois(canvas) {
             validate_component(roi_name, "ROI name")?;
             let crop = normalized.crop_imm(roi.x1, roi.y1, roi.width(), roi.height());
             let relative_path = PathBuf::from("images")
@@ -164,7 +164,7 @@ fn crop_batch(args: &[String]) -> Result<String, String> {
             source_file: case.image,
             source_sha256,
             source_size,
-            normalized_size: manifest.standard_size,
+            normalized_size: canvas,
             quality,
             rois,
         });

rust/ocrkit-image-core/src/lib.rs

@@ -18,6 +18,30 @@ impl ImageSize {
     }
 }
 
+/// 3440x1440; the 3% tolerance also covers 21:9, 2560x1080 and 3840x1600.
+const ULTRAWIDE_ASPECT: f64 = 43.0 / 18.0;
+const ASPECT_TOLERANCE: f64 = 0.03;
+
+fn relative_aspect_difference(aspect_ratio: f64, target_ratio: f64) -> f64 {
+    (aspect_ratio - target_ratio).abs() / target_ratio
+}
+
+pub fn is_ultrawide(source: ImageSize) -> bool {
+    source.height > 0
+        && relative_aspect_difference(source.width as f64 / source.height as f64, ULTRAWIDE_ASPECT)
+            <= ASPECT_TOLERANCE
+}
+
+/// The canvas a source is normalized onto: ultrawide sources keep the standard height scale.
+pub fn canvas_size(source: ImageSize, standard: ImageSize) -> ImageSize {
+    if !is_ultrawide(source) {
+        return standard;
+    }
+    let numerator = standard.height as u64 * source.width as u64;
+    let width = (numerator * 2 + source.height as u64) / (2 * source.height as u64);
+    ImageSize::new(width as u32, standard.height)
+}
+
 #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct RoiBox {
     pub x1: u32,
@@ -39,6 +63,21 @@ impl RoiBox {
         self.y2.saturating_sub(self.y1)
     }
 
+    /// Place the ROI on a canvas wider than the standard one: left ROIs stay put, right ROIs
+    /// follow the right edge and central ROIs the middle.
+    pub fn anchored(self, standard: ImageSize, canvas_width: u32) -> Self {
+        let extra = canvas_width.saturating_sub(standard.width);
+        let doubled_centre = (self.x1 + self.x2) as u64 * 3;
+        let shift = if doubled_centre < 2 * standard.width as u64 {
+            0
+        } else if doubled_centre > 4 * standard.width as u64 {
+            extra
+        } else {
+            extra / 2
+        };
+        Self::new(self.x1 + shift, self.y1, self.x2 + shift, self.y2)
+    }
+
     pub fn scaled_to(self, source: ImageSize, standard: ImageSize) -> Result<Self, LayoutError> {
         if source.width == 0 || source.height == 0 {
             return Err(LayoutError::InvalidImageSize(source));
@@ -94,17 +133,22 @@ impl LayoutManifest {
         Ok(())
     }
 
+    pub fn anchored_rois(&self, canvas: ImageSize) -> BTreeMap<String, RoiBox> {
+        self.rois
+            .iter()
+            .map(|(name, roi)| (name.clone(), roi.anchored(self.standard_size, canvas.width)))
+            .collect()
+    }
+
     pub fn scaled_rois(&self, source: ImageSize) -> Result<BTreeMap<String, RoiBox>, LayoutError> {
         self.validate()?;
         if source.width == 0 || source.height == 0 {
             return Err(LayoutError::InvalidImageSize(source));
         }
-        self.rois
-            .iter()
-            .map(|(name, roi)| {
-                roi.scaled_to(source, self.standard_size)
-                    .map(|scaled| (name.clone(), scaled))
-            })
+        let canvas = canvas_size(source, self.standard_size);
+        self.anchored_rois(canvas)
+            .into_iter()
+            .map(|(name, roi)| roi.scaled_to(source, canvas).map(|scaled| (name, scaled)))
             .collect()
     }
 }
@@ -127,11 +171,14 @@ pub fn assess_quality(source: ImageSize, target: ImageSize) -> Result<QualityRep
     }
 
     let aspect_ratio = source.width as f64 / source.height as f64;
-    let target_ratio = target.width as f64 / target.height as f64;
-    let aspect_difference = (aspect_ratio - target_ratio).abs() / target_ratio;
-    let tolerance = 0.03;
-    let cropped = aspect_difference > tolerance;
-    let layout_confidence = (1.0 - aspect_difference / tolerance).clamp(0.0, 1.0);
+    let target_ratio = if is_ultrawide(source) {
+        ULTRAWIDE_ASPECT
+    } else {
+        target.width as f64 / target.height as f64
+    };
+    let aspect_difference = relative_aspect_difference(aspect_ratio, target_ratio);
+    let cropped = aspect_difference > ASPECT_TOLERANCE;
+    let layout_confidence = (1.0 - aspect_difference / ASPECT_TOLERANCE).clamp(0.0, 1.0);
     let warnings = if cropped {
         vec![
             "quality.aspect_ratio_mismatch".to_string(),
@@ -234,6 +281,53 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn ultrawide_keeps_height_scale_and_anchors_rois_to_screen_edges() {
+        let mut layout = manifest();
+        layout
+            .rois
+            .insert("left_panel".to_string(), RoiBox::new(35, 15, 220, 225));
+        layout.rois.insert(
+            "center_banner".to_string(),
+            RoiBox::new(150, 125, 1160, 215),
+        );
+        let source = ImageSize::new(3440, 1440);
+
+        assert_eq!(
+            canvas_size(source, layout.standard_size),
+            ImageSize::new(1720, 720)
+        );
+        assert_eq!(
+            canvas_size(ImageSize::new(2560, 1440), layout.standard_size),
+            layout.standard_size
+        );
+        let anchored = layout.anchored_rois(ImageSize::new(1720, 720));
+        assert_eq!(anchored["left_panel"], RoiBox::new(35, 15, 220, 225));
+        assert_eq!(anchored["center_banner"], RoiBox::new(370, 125, 1380, 215));
+        assert_eq!(anchored["right_panel"], RoiBox::new(1480, 0, 1715, 105));
+        assert_eq!(
+            layout.scaled_rois(source).unwrap()["right_panel"],
+            RoiBox::new(2960, 0, 3430, 210)
+        );
+    }
+
+    #[test]
+    fn ultrawide_is_not_flagged_as_cropped() {
+        for (width, height) in [(3440, 1440), (2560, 1080), (3840, 1600)] {
+            let quality =
+                assess_quality(ImageSize::new(width, height), ImageSize::new(1280, 720)).unwrap();
+            assert!(
+                !quality.cropped && quality.warnings.is_empty(),
+                "{width}x{height}"
+            );
+        }
+        assert!(
+            assess_quality(ImageSize::new(2000, 1000), ImageSize::new(1280, 720))
+                .unwrap()
+                .cropped
+        );
+    }
+
     #[test]
     fn quality_matches_python_aspect_contract() {
         let quality =

tests/test_image_quality.py

@@ -16,6 +16,14 @@ def test_quality_preserves_original_size_and_reports_supported_layout() -> None:
     assert quality["warnings"] == []
 
 
+def test_quality_accepts_ultrawide_but_still_flags_other_wide_crops() -> None:
+    for width, height in ((3440, 1440), (2560, 1080), (3840, 1600)):
+        quality = assess_input_quality(np.zeros((height, width, 3), dtype=np.uint8), 1280, 720)
+        assert quality["cropped"] is False, (width, height)
+        assert quality["warnings"] == []
+    assert assess_input_quality(np.zeros((1000, 2000, 3), dtype=np.uint8), 1280, 720)["cropped"] is True
+
+
 def test_quality_reports_aspect_ratio_risk_before_normalization() -> None:
     image = np.zeros((1000, 1500, 3), dtype=np.uint8)
 

tests/test_roi_selection.py

@@ -1,7 +1,9 @@
+from pathlib import Path
+
 import numpy as np
 
-from app.core.roi_config import RoiBox, RoiConfig
-from app.image.roi import select_roi_config
+from app.core.roi_config import RoiBox, RoiConfig, load_roi_config
+from app.image.roi import anchored_rois, crop_all_rois, select_roi_config
 
 
 def _config(width: int, height: int, version: str) -> RoiConfig:
@@ -14,3 +16,19 @@ def test_select_roi_config_uses_matching_source_aspect_ratio() -> None:
 
     assert select_roi_config(np.zeros((1600, 2560, 3), dtype=np.uint8), (widescreen, tall)) == tall
     assert select_roi_config(np.zeros((1440, 2560, 3), dtype=np.uint8), (widescreen, tall)) == widescreen
+
+
+def test_ultrawide_keeps_height_scale_and_anchors_hud_to_screen_edges() -> None:
+    config = load_roi_config(Path("configs/roi_1280x720.yaml"))
+    rng = np.random.default_rng(7)
+    standard = rng.integers(0, 256, (1440, 2560, 3), dtype=np.uint8)
+    extra = rng.integers(0, 256, (1440, 880, 3), dtype=np.uint8)
+    ultrawide = np.concatenate([standard[:, :1280], extra, standard[:, 1280:]], axis=1)
+
+    _, standard_crops = crop_all_rois(standard, config)
+    normalized, wide_crops = crop_all_rois(ultrawide, config)
+
+    assert normalized.shape[:2] == (720, 1720)
+    for name in ("left_panel", "run_code_panel", "achievement_panel", "bottom_left_hero", "right_panel", "run_code_right_panel", "ai_watermark", "server_load_panel"):
+        assert np.array_equal(wide_crops[name], standard_crops[name]), name
+    assert anchored_rois(config, 1720)["right_panel"].x2 == config.rois["right_panel"].x2 + 440