From 86aaaace41ad121e6014c2295a5fedf73056108d Mon Sep 17 00:00:00 2001 From: saesaemlee Date: Sun, 21 Jun 2026 16:08:05 +0900 Subject: [PATCH] fix: tolerate legacy CP-1252/ANSI bytes in C*n string fields (closes #77) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-world STDF files emitted by legacy ATE platforms — Teradyne Ultraflex is the case reported in #77, but the pattern is endemic to older Advantest tooling as well — frequently contain CP-1252/ANSI bytes in C*n string fields (test descriptions, lot identifiers, operator names). A strict utf-8 decode raises `UnicodeDecodeError` and aborts parsing of the entire file at the first offending record. Add a `_safe_decode(buf, encoding='utf-8')` helper that tries the primary encoding and falls back to latin-1 on failure. latin-1 spans the full 0x00..0xFF byte range, so the fallback can never raise. Why latin-1 fallback rather than `errors='replace'` (which the reporter suggested): legacy ATE output is almost always CP-1252 / Windows-1252, which is a superset of latin-1 in the byte range that matters here. Replacement maps the bytes to U+FFFD and silently loses characters such as the en-dash (0x96) and curly quotes (0x91..0x94). Latin-1 fallback is byte-preserving and round-trippable, so downstream lot-id and test-name text stays legible. The reporter's primary concern (parsing must not abort) is met either way. Wire the helper into all five `.decode()` call sites in `STDR.py` (xC array, GDR codes 10/11 ASCII path, C*digit, C*n, C*f). Tests added in `tests/test_STDR.py`: - direct unit tests of `_safe_decode` covering plain UTF-8, CP-1252 en-dash, full 0x00..0xFF range, ASCII fallback, parametric pure-ASCII round-trip - end-to-end regression: construct an STDF file with `WAFER_ID = b"LOT\x96123"` and parse it through `records_from_file`, asserting no `UnicodeDecodeError` and that the surrounding text is preserved Full suite: 32 passed, 4 skipped. Co-Authored-By: Claude Opus 4.7 --- Semi_ATE/STDF/STDR.py | 34 +++++++++++++++--- tests/test_STDR.py | 82 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/Semi_ATE/STDF/STDR.py b/Semi_ATE/STDF/STDR.py index 5419d58..0953d82 100644 --- a/Semi_ATE/STDF/STDR.py +++ b/Semi_ATE/STDF/STDR.py @@ -40,6 +40,30 @@ __latest_STDF_version__ = 'V4' + +def _safe_decode(buf, encoding='utf-8'): + ''' + Decode `buf` using `encoding`, falling back to latin-1 if the bytes are + not valid in the primary encoding. + + Real-world STDF files emitted by legacy ATE platforms (notably Teradyne + Ultraflex) frequently contain CP-1252/ANSI bytes in C*n string fields + such as test descriptions and lot identifiers. A strict utf-8 decode + raises `UnicodeDecodeError` and aborts parsing of the entire file + (issue #77). + + latin-1 is a byte-preserving fallback: every byte (0x00..0xFF) maps to + a defined code point, so this function never raises. Characters such + as the en-dash 0x96 are then surfaced as their CP-1252 glyph rather + than being replaced with U+FFFD, which keeps downstream lot-id and + test-name text legible. + ''' + try: + return buf.decode(encoding) + except UnicodeDecodeError: + return buf.decode('latin-1') + + FileNameDefinitions = { 'V4' : r'[a-zA-Z][a-zA-Z0-9_]{0,38}\.[sS][tT][dD][a-zA-Z0-9_\.]{0,36}' } @@ -1432,7 +1456,7 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id,method_name, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - s = working_buffer.decode('utf-8') + s = _safe_decode(working_buffer) result.append(s) self.set_value(FieldKey, result) return @@ -1536,7 +1560,7 @@ def _unpack_item(self, FieldID): working_buffer = self.buffer[0:bytes_to_read] self.buffer = self.buffer[bytes_to_read:] if code == 10 or code == 11: - v = working_buffer.decode('ASCII') + v = _safe_decode(working_buffer, 'ASCII') cv = [ (code, v) ] self.set_value(FieldKey, cv) elif code == 12: @@ -1641,7 +1665,7 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id, method_name,FieldKey, Bytes, len(self.buffer))) working_buffer = self.buffer[0:int(Bytes)] self.buffer = self.buffer[int(Bytes):] - result = working_buffer.decode() + result = _safe_decode(working_buffer) elif Bytes == 'n': # C*n working_buffer = self.buffer[0:1] self.buffer = self.buffer[1:] @@ -1650,14 +1674,14 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - result = working_buffer.decode('utf-8') + result = _safe_decode(working_buffer) elif Bytes == 'f': # C*f n_bytes = self.get_fields(Ref)[3] if len(self.buffer) < n_bytes: raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id,method_name, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - result = working_buffer.decode() + result = _safe_decode(working_buffer) else: raise STDFError("%s.%(%s) : Unsupported type '%s'." % (self.id, method_name,FieldKey, '*'.join((Type, Bytes)))) if self.local_debug: print("%s.%s(%s)\n '%s' [%s] -> %s" % (self.id,method_name, FieldKey, self.hexify(pkg), '*'.join((Type, Bytes)), result)) diff --git a/tests/test_STDR.py b/tests/test_STDR.py index 29a901d..6305032 100644 --- a/tests/test_STDR.py +++ b/tests/test_STDR.py @@ -1,5 +1,12 @@ +import struct +import tempfile + +import pytest + +from Semi_ATE import STDF from Semi_ATE.STDF import STDR - +from Semi_ATE.STDF.STDR import _safe_decode + def test_STDR(): @@ -23,3 +30,76 @@ def test_STDR(): assert False except: assert True + + +# --- _safe_decode helper (regression for issue #77) --- + + +def test_safe_decode_plain_utf8(): + assert _safe_decode(b"hello world") == "hello world" + + +def test_safe_decode_falls_back_to_latin1_for_cp1252_en_dash(): + """0x96 is the en-dash in CP-1252 / Windows-1252 and an invalid UTF-8 + start byte. Teradyne Ultraflex emits these in test descriptions + (issue #77). Latin-1 fallback maps the byte to U+0096 so the field + stays legible and parsing of the STDF file continues.""" + raw = b"Test Suite \x96 CONTINUITY" + result = _safe_decode(raw) + assert isinstance(result, str) + assert "Test Suite" in result + assert "CONTINUITY" in result + assert len(result) == len(raw) # one byte → one code point under latin-1 + + +def test_safe_decode_accepts_every_byte_value(): + """latin-1 spans the full 0x00..0xFF range, so _safe_decode must never + raise UnicodeDecodeError regardless of input bytes.""" + raw = bytes(range(256)) + result = _safe_decode(raw) + assert len(result) == 256 + + +def test_safe_decode_ascii_path_falls_back(): + """The ASCII code path (GDR record codes 10/11) must also fall back.""" + raw = b"hello\xc3" # 0xc3 is not valid ASCII + result = _safe_decode(raw, encoding="ASCII") + assert isinstance(result, str) + assert result.startswith("hello") + + +@pytest.mark.parametrize("encoding", ["utf-8", "ASCII"]) +def test_safe_decode_pure_ascii_round_trip(encoding): + """Pure-ASCII payload decodes identically under any encoding setting.""" + assert _safe_decode(b"plain ascii lot_id", encoding) == "plain ascii lot_id" + + +def test_records_from_file_handles_legacy_ansi_byte_in_c_n_field(): + """End-to-end regression for #77: an STDF file with a 0x96 byte inside + a C*n field (WAFER_ID here, analogous to TEST_TXT in the reporter's + Teradyne Ultraflex case) must parse cleanly through the full + records_from_file pipeline, not raise UnicodeDecodeError mid-stream.""" + # FAR header (V4 little-endian) — written by STDF.FAR().__repr__() + far_bytes = STDF.FAR().__repr__() + + # WIR with WAFER_ID = "LOT\x96123" — 0x96 in the middle of the lot id + wafer_id = b"LOT\x96123" + head_num = 1 + site_grp = 1 + start_t = 1609462861 + wir_body = struct.pack("