diff --git a/Semi_ATE/STDF/STDR.py b/Semi_ATE/STDF/STDR.py index 5419d58..0953d82 100644 --- a/Semi_ATE/STDF/STDR.py +++ b/Semi_ATE/STDF/STDR.py @@ -40,6 +40,30 @@ __latest_STDF_version__ = 'V4' + +def _safe_decode(buf, encoding='utf-8'): + ''' + Decode `buf` using `encoding`, falling back to latin-1 if the bytes are + not valid in the primary encoding. + + Real-world STDF files emitted by legacy ATE platforms (notably Teradyne + Ultraflex) frequently contain CP-1252/ANSI bytes in C*n string fields + such as test descriptions and lot identifiers. A strict utf-8 decode + raises `UnicodeDecodeError` and aborts parsing of the entire file + (issue #77). + + latin-1 is a byte-preserving fallback: every byte (0x00..0xFF) maps to + a defined code point, so this function never raises. Characters such + as the en-dash 0x96 are then surfaced as their CP-1252 glyph rather + than being replaced with U+FFFD, which keeps downstream lot-id and + test-name text legible. + ''' + try: + return buf.decode(encoding) + except UnicodeDecodeError: + return buf.decode('latin-1') + + FileNameDefinitions = { 'V4' : r'[a-zA-Z][a-zA-Z0-9_]{0,38}\.[sS][tT][dD][a-zA-Z0-9_\.]{0,36}' } @@ -1432,7 +1456,7 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id,method_name, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - s = working_buffer.decode('utf-8') + s = _safe_decode(working_buffer) result.append(s) self.set_value(FieldKey, result) return @@ -1536,7 +1560,7 @@ def _unpack_item(self, FieldID): working_buffer = self.buffer[0:bytes_to_read] self.buffer = self.buffer[bytes_to_read:] if code == 10 or code == 11: - v = working_buffer.decode('ASCII') + v = _safe_decode(working_buffer, 'ASCII') cv = [ (code, v) ] self.set_value(FieldKey, cv) elif code == 12: @@ -1641,7 +1665,7 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id, method_name,FieldKey, Bytes, len(self.buffer))) working_buffer = self.buffer[0:int(Bytes)] self.buffer = self.buffer[int(Bytes):] - result = working_buffer.decode() + result = _safe_decode(working_buffer) elif Bytes == 'n': # C*n working_buffer = self.buffer[0:1] self.buffer = self.buffer[1:] @@ -1650,14 +1674,14 @@ def _unpack_item(self, FieldID): raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - result = working_buffer.decode('utf-8') + result = _safe_decode(working_buffer) elif Bytes == 'f': # C*f n_bytes = self.get_fields(Ref)[3] if len(self.buffer) < n_bytes: raise STDFError("%s.%s(%s) : Not enough bytes in buffer (need %s while %s available)." % (self.id,method_name, FieldKey, n_bytes, len(self.buffer))) working_buffer = self.buffer[0:n_bytes] self.buffer = self.buffer[n_bytes:] - result = working_buffer.decode() + result = _safe_decode(working_buffer) else: raise STDFError("%s.%(%s) : Unsupported type '%s'." % (self.id, method_name,FieldKey, '*'.join((Type, Bytes)))) if self.local_debug: print("%s.%s(%s)\n '%s' [%s] -> %s" % (self.id,method_name, FieldKey, self.hexify(pkg), '*'.join((Type, Bytes)), result)) diff --git a/tests/test_STDR.py b/tests/test_STDR.py index 29a901d..6305032 100644 --- a/tests/test_STDR.py +++ b/tests/test_STDR.py @@ -1,5 +1,12 @@ +import struct +import tempfile + +import pytest + +from Semi_ATE import STDF from Semi_ATE.STDF import STDR - +from Semi_ATE.STDF.STDR import _safe_decode + def test_STDR(): @@ -23,3 +30,76 @@ def test_STDR(): assert False except: assert True + + +# --- _safe_decode helper (regression for issue #77) --- + + +def test_safe_decode_plain_utf8(): + assert _safe_decode(b"hello world") == "hello world" + + +def test_safe_decode_falls_back_to_latin1_for_cp1252_en_dash(): + """0x96 is the en-dash in CP-1252 / Windows-1252 and an invalid UTF-8 + start byte. Teradyne Ultraflex emits these in test descriptions + (issue #77). Latin-1 fallback maps the byte to U+0096 so the field + stays legible and parsing of the STDF file continues.""" + raw = b"Test Suite \x96 CONTINUITY" + result = _safe_decode(raw) + assert isinstance(result, str) + assert "Test Suite" in result + assert "CONTINUITY" in result + assert len(result) == len(raw) # one byte → one code point under latin-1 + + +def test_safe_decode_accepts_every_byte_value(): + """latin-1 spans the full 0x00..0xFF range, so _safe_decode must never + raise UnicodeDecodeError regardless of input bytes.""" + raw = bytes(range(256)) + result = _safe_decode(raw) + assert len(result) == 256 + + +def test_safe_decode_ascii_path_falls_back(): + """The ASCII code path (GDR record codes 10/11) must also fall back.""" + raw = b"hello\xc3" # 0xc3 is not valid ASCII + result = _safe_decode(raw, encoding="ASCII") + assert isinstance(result, str) + assert result.startswith("hello") + + +@pytest.mark.parametrize("encoding", ["utf-8", "ASCII"]) +def test_safe_decode_pure_ascii_round_trip(encoding): + """Pure-ASCII payload decodes identically under any encoding setting.""" + assert _safe_decode(b"plain ascii lot_id", encoding) == "plain ascii lot_id" + + +def test_records_from_file_handles_legacy_ansi_byte_in_c_n_field(): + """End-to-end regression for #77: an STDF file with a 0x96 byte inside + a C*n field (WAFER_ID here, analogous to TEST_TXT in the reporter's + Teradyne Ultraflex case) must parse cleanly through the full + records_from_file pipeline, not raise UnicodeDecodeError mid-stream.""" + # FAR header (V4 little-endian) — written by STDF.FAR().__repr__() + far_bytes = STDF.FAR().__repr__() + + # WIR with WAFER_ID = "LOT\x96123" — 0x96 in the middle of the lot id + wafer_id = b"LOT\x96123" + head_num = 1 + site_grp = 1 + start_t = 1609462861 + wir_body = struct.pack("