diff --git a/Directory.Packages.props b/Directory.Packages.props index 5905c11..dced9c5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -56,6 +56,7 @@ + diff --git a/src/CouponHubBot/CouponHubBot.fsproj b/src/CouponHubBot/CouponHubBot.fsproj index aef2c3c..4933c94 100644 --- a/src/CouponHubBot/CouponHubBot.fsproj +++ b/src/CouponHubBot/CouponHubBot.fsproj @@ -27,6 +27,7 @@ + diff --git a/src/CouponHubBot/Services/CouponOcrEngine.fs b/src/CouponHubBot/Services/CouponOcrEngine.fs index 3743e9e..71811de 100644 --- a/src/CouponHubBot/Services/CouponOcrEngine.fs +++ b/src/CouponHubBot/Services/CouponOcrEngine.fs @@ -210,8 +210,71 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti let someMoney (v: decimal) : Nullable = Nullable(v) let someDate (v: DateTime) : Nullable = Nullable(v) - let tryDecodeBarcode (imageBytes: ReadOnlyMemory) = - use activity = botActivity.StartActivity("couponOcr.barcode") + let tryDecodeNative (imageBytes: ReadOnlyMemory) = + use activity = botActivity.StartActivity("couponOcr.barcode.native") + try + // Resolve the library before constructing objects with native finalizers. + let format = ZXingCpp.BarcodeFormat.Parse("EAN13") + use ms = new MemoryStream(imageBytes.ToArray()) + use image = Image.Load(ms) + let pixels = Array.zeroCreate (image.Width * image.Height) + image.CopyPixelDataTo(pixels) + use reader = new ZXingCpp.BarcodeReader() + reader.Formats <- ZXingCpp.BarcodeFormats(format) + reader.TryHarder <- true + reader.TryRotate <- true + reader.TryInvert <- false + reader.TryDownscale <- true + reader.MaxNumberOfSymbols <- 1 + reader.MinLineCount <- 2 + reader.ReturnErrors <- false + // ImageView borrows the pixel buffer throughout native decoding. + use _pin = Memory(pixels).Pin() + let decode x y width height = + let offset = y * image.Width + x + let length = (height - 1) * image.Width + width + let region = ReadOnlySpan(pixels, offset, length) + let view = ZXingCpp.ImageView(region, width, height, ZXingCpp.ImageFormat.Lum, image.Width) + let results = + try reader.From(view) + finally GC.KeepAlive(view) + try + results + |> Array.tryPick (fun barcode -> + if barcode.IsValid && not (String.IsNullOrWhiteSpace barcode.Text) then + Some barcode.Text + else + None) + finally + for barcode in results do + barcode.Dispose() + + let result = + match decode 0 0 image.Width image.Height with + | Some text -> Some text + | None -> + let bands = [| 0.0, 0.50; 0.0, 0.60; 0.20, 0.70; 0.30, 0.80; 0.45, 1.0; 0.55, 1.0; 0.65, 1.0 |] + [| 0.0; 0.10; 0.20 |] + |> Array.tryPick (fun margin -> + bands + |> Array.tryPick (fun (top, bottom) -> + let x = int (Math.Round(float image.Width * margin)) + let y = min (image.Height - 1) (int (Math.Round(float image.Height * top))) + let width = image.Width - 2 * x + let height = max 1 (int (Math.Round(float image.Height * bottom)) - y) + decode x y width height)) + if not (isNull activity) then + %activity.SetTag("barcode.found", result.IsSome) + result + with ex -> + if not (isNull activity) then + %activity.SetTag("error.type", ex.GetType().FullName) + %activity.SetStatus(ActivityStatusCode.Error) + logger.LogWarning(ex, "Native barcode decode failed; trying managed decoder") + None + + let tryDecodeManaged (imageBytes: ReadOnlyMemory) = + use activity = botActivity.StartActivity("couponOcr.barcode.managed") let mutable attempts = 0 try use original = @@ -224,12 +287,12 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti let opts = DecodingOptions() opts.TryHarder <- true opts.PossibleFormats <- [| BarcodeFormat.EAN_13 |] - opts.TryInverted <- true + opts.TryInverted <- false let reader = BarcodeReader() reader.Options <- opts reader.AutoRotate <- true - let decode (label: string) (img: Image) = + let decode (label: string) (img: LuminanceSource) = use attempt = botActivity.StartActivity("couponOcr.barcode.decode") attempts <- attempts + 1 if not (isNull attempt) then @@ -248,12 +311,12 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti let clamp (v: int) (minv: int) (maxv: int) = max minv (min maxv v) - let cropTry (baseLabel: string) (img: Image) (rect: Rectangle) = + let cropTry (baseLabel: string) (img: LuminanceSource) (rect: Rectangle) = let x = clamp rect.X 0 (img.Width - 1) let y = clamp rect.Y 0 (img.Height - 1) let w = clamp rect.Width 1 (img.Width - x) let h = clamp rect.Height 1 (img.Height - y) - use cropped = img.Clone(fun ctx -> ctx.Crop(Rectangle(x, y, w, h)) |> ignore) + let cropped = img.crop(x, y, w, h) decode baseLabel cropped let tryOnImage (labelPrefix: string) (img: Image) = @@ -263,7 +326,8 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti // - Middle band crops (covers centre region) // - Bottom crops (useful for app screenshots) // - Center-narrowed variants of the above (avoid noise on sides) - match decode (labelPrefix + ":full") img with + let luminance = ImageSharpLuminanceSource(img) + match decode (labelPrefix + ":full") luminance with | Some t -> Some t | None -> let w = img.Width @@ -312,7 +376,7 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti allRects |> Array.mapi (fun i r -> (i, r)) - |> Array.tryPick (fun (i, r) -> cropTry $"{labelPrefix}:crop{i}" img r) + |> Array.tryPick (fun (i, r) -> cropTry $"{labelPrefix}:crop{i}" luminance r) let tryAll () = match tryOnImage "orig" original with @@ -369,6 +433,16 @@ type CouponOcrEngine(azureTextOcr: IBotOcr, logger: ILogger, ti logger.LogWarning(ex, "Barcode decode failed") noneBarcode + let tryDecodeBarcode (imageBytes: ReadOnlyMemory) = + use activity = botActivity.StartActivity("couponOcr.barcode") + let result = + match tryDecodeNative imageBytes with + | Some text -> text + | None -> tryDecodeManaged imageBytes + if not (isNull activity) then + %activity.SetTag("barcode.found", not (String.IsNullOrWhiteSpace result)) + result + let parseFromText (nowUtc: DateTime) (ocrText: string) = let amounts = CouponOcrParsing.findEuroAmounts ocrText let dates = CouponOcrParsing.findDates nowUtc ocrText diff --git a/src/CouponHubBot/docs/OCR.md b/src/CouponHubBot/docs/OCR.md index 1dcad07..f515b48 100644 --- a/src/CouponHubBot/docs/OCR.md +++ b/src/CouponHubBot/docs/OCR.md @@ -1,27 +1,30 @@ -# OCR Pipeline - -## Overview - -`CouponOcrEngine` combines barcode scanning (ZXing) with Azure Computer Vision text OCR to extract coupon data from photos. - -## Components - -- **ZXing.Net** + ImageSharp bindings — barcode/QR code scanning -- **AzureOcrService** — calls Azure Computer Vision `imageanalysis:analyze` endpoint -- **CouponOcrEngine** — orchestrates both, returns `CouponOCR` record - -## OCR Result - -`CouponOCR` contains optional fields: `value`, `min_check`, `expires_at`, `barcode_text`. Any combination may be present. The result is **always a pre-fill suggestion** — the user confirms final values in the wizard. - -## Caching (for tests) - -`tests/CouponHubBot.Ocr.Tests/AzureCache/` stores Azure OCR responses as `*.azure.json` files. These are git-tracked so tests can run without Azure API keys (cache-only mode). - -## Configuration - -Env vars: -- `OCR_ENABLED` — enable/disable OCR (default: false) -- `OCR_MAX_FILE_SIZE_BYTES` — max photo size for OCR processing -- `AZURE_OCR_ENDPOINT` — Azure Computer Vision endpoint URL -- `AZURE_OCR_KEY` — Azure API key +# OCR Pipeline + +## Overview + +`CouponOcrEngine` combines barcode scanning (ZXing) with Azure Computer Vision text OCR to extract coupon data from photos. + +## Components + +- **ZXingCpp** + ImageSharp — EAN-13 scanning on grayscale images, followed by horizontal crops +- **ZXing.Net** + ImageSharp bindings — fallback with rotation, contrast and resizing; luminance is shared across crops +- **AzureOcrService** — calls Azure Computer Vision `imageanalysis:analyze` endpoint +- **CouponOcrEngine** — orchestrates both, returns `CouponOCR` record + +Native library load or decode failures fall back to the managed decoder. The Linux native binaries require glibc 2.38 and libstdc++ with GLIBCXX_3.4.31; the Noble runtime image provides both. + +## OCR Result + +`CouponOCR` contains optional fields: `value`, `min_check`, `expires_at`, `barcode_text`. Any combination may be present. The result is **always a pre-fill suggestion** — the user confirms final values in the wizard. + +## Caching (for tests) + +`tests/CouponHubBot.Ocr.Tests/AzureCache/` stores Azure OCR responses as `*.azure.json` files. These are git-tracked so tests can run without Azure API keys (cache-only mode). + +## Configuration + +Env vars: +- `OCR_ENABLED` — enable/disable OCR (default: false) +- `OCR_MAX_FILE_SIZE_BYTES` — max photo size for OCR processing +- `AZURE_OCR_ENDPOINT` — Azure Computer Vision endpoint URL +- `AZURE_OCR_KEY` — Azure API key diff --git a/tests/BotInfra.Tests/EventStoreSnapshotTests.fs b/tests/BotInfra.Tests/EventStoreSnapshotTests.fs index a83c923..709ba1f 100644 --- a/tests/BotInfra.Tests/EventStoreSnapshotTests.fs +++ b/tests/BotInfra.Tests/EventStoreSnapshotTests.fs @@ -59,16 +59,16 @@ let private hasEvent (name: string) (a: Activity) = a.Events |> Seq.exists (fun /// A fold that parks its first call until released — holds a load between its query and its cache put. let private parkedFold () = - let entered = new SemaphoreSlim(0) + let entered = TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) let release = new SemaphoreSlim(0) let mutable first = true let f s e = if first then first <- false - %entered.Release() - release.Wait() + entered.SetResult() + Assert.True(release.Wait(TimeSpan.FromSeconds 30.0), "The concurrent operation did not release the parked fold") fold s e - f, entered, release + f, entered.Task, release type SnapshotRow = { schema_version: int; stream_version: int; state: string } @@ -448,10 +448,13 @@ type EventStoreSnapshotTests(db: PostgresFixture) = do! appendRaw sid (added 2) use _scope = store.BeginRequestScope() let parked, entered, release = parkedFold () - let slowLoad = store.LoadState(parked, Tally.Zero, policy 100, sid) - do! entered.WaitAsync() - let! _ = increment (policy 100) sid 10 - %release.Release() + let slowLoad = Task.Run(fun () -> store.LoadState(parked, Tally.Zero, policy 100, sid)) + try + do! entered.WaitAsync(TimeSpan.FromSeconds 30.0) + let! _ = increment (policy 100) sid 10 + () + finally + %release.Release() let! (_, slowVersion) = slowLoad Assert.Equal(2, slowVersion) let! (state, version) = load (policy 100) sid @@ -465,10 +468,13 @@ type EventStoreSnapshotTests(db: PostgresFixture) = do! appendRaw sid (added 2) use _scope = store.BeginRequestScope() let parked, entered, release = parkedFold () - let slowLoad = store.LoadState(parked, Tally.Zero, policy 100, sid) - do! entered.WaitAsync() - let! _ = store.Transact(fold, Tally.Zero, (fun _ -> [ Added {| amount = 10 |} ]), sid) - %release.Release() + let slowLoad = Task.Run(fun () -> store.LoadState(parked, Tally.Zero, policy 100, sid)) + try + do! entered.WaitAsync(TimeSpan.FromSeconds 30.0) + let! _ = store.Transact(fold, Tally.Zero, (fun _ -> [ Added {| amount = 10 |} ]), sid) + () + finally + %release.Release() let! _ = slowLoad let! (state, version) = load (policy 100) sid let! (_, rawVersion) = store.GetRawEventsForStream sid @@ -504,11 +510,14 @@ type EventStoreSnapshotTests(db: PostgresFixture) = let otherPod = EventStore(db.ConnectionString, "event", jsonOpts, "event_snapshot") use spans = new LoadSpans() let parked, entered, release = parkedFold () - let slowLoad = store.LoadState(parked, Tally.Zero, policy 1, sid) - do! entered.WaitAsync() - do! appendRaw sid (added 3) - let! _ = otherPod.LoadState(fold, Tally.Zero, policy 1, sid) - %release.Release() + let slowLoad = Task.Run(fun () -> store.LoadState(parked, Tally.Zero, policy 1, sid)) + try + do! entered.WaitAsync(TimeSpan.FromSeconds 30.0) + do! appendRaw sid (added 3) + let! _ = otherPod.LoadState(fold, Tally.Zero, policy 1, sid) + () + finally + %release.Release() let! (_, slowVersion) = slowLoad Assert.Equal(2, slowVersion) diff --git a/tests/CouponHubBot.Ocr.Tests/OcrTests.fs b/tests/CouponHubBot.Ocr.Tests/OcrTests.fs index ad35468..0460b74 100644 --- a/tests/CouponHubBot.Ocr.Tests/OcrTests.fs +++ b/tests/CouponHubBot.Ocr.Tests/OcrTests.fs @@ -8,10 +8,14 @@ open System.Net.Http open System.Text open Microsoft.Extensions.Logging open Microsoft.Extensions.Options +open SixLabors.ImageSharp +open SixLabors.ImageSharp.PixelFormats +open SixLabors.ImageSharp.Processing open Xunit open CouponHubBot open CouponHubBot.Services +open CouponHubBot.Utils open BotInfra module private XUnitLogging = @@ -179,6 +183,12 @@ module private Parsing = type OcrTests(output: ITestOutputHelper) = + let buildCpuOnlyEngine () = + let azure = + { new IBotOcr with + member _.AnalyzeImageBytes _ = task { return null } } + CouponOcrEngine(azure, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, TimeProvider.System) + let buildEngine (imageFileName: string) = let logs = ResizeArray() let endpoint = Environment.GetEnvironmentVariable("AZURE_OCR_ENDPOINT") @@ -213,6 +223,34 @@ type OcrTests(output: ITestOutputHelper) = CouponOcrEngine(azure, XUnitLogging.XUnitLogger(output, logs), timeProvider) engine, http, logs + [] + [] + [] + [] + [] + member _.``Coupon barcode is recognized without OCR at any right angle``(angle: int) = + task { + let path = Path.Combine(AppContext.BaseDirectory, "Images", "10_50_01-04_01-13_2706602781191.jpg") + use image = Image.Load(path) + image.Mutate(fun ctx -> %ctx.Rotate(float32 angle)) + use stream = new MemoryStream() + image.SaveAsPng(stream) + let! result = (buildCpuOnlyEngine ()).Recognize(ReadOnlyMemory(stream.ToArray())) + Assert.Equal("2706602781191", result.barcode) + } + + [] + [] + [] + member _.``Blank image does not invent a barcode``(width: int, height: int) = + task { + use image = new Image(width, height, L8(255uy)) + use stream = new MemoryStream() + image.SaveAsPng(stream) + let! result = (buildCpuOnlyEngine ()).Recognize(ReadOnlyMemory(stream.ToArray())) + Assert.Null(result.barcode) + } + [] [] []