diff --git a/crates/zz_cli/tests/dual_engine_parity.rs b/crates/zz_cli/tests/dual_engine_parity.rs index 8a2f187d..9a5a28fb 100644 --- a/crates/zz_cli/tests/dual_engine_parity.rs +++ b/crates/zz_cli/tests/dual_engine_parity.rs @@ -1136,6 +1136,7 @@ parity_strict!( "vec_nested_str_parity.zz" ); parity_strict!(parity_stdlib_str_find_test, "stdlib", "str_find_test.zz"); +parity_strict!(parity_stdlib_str_bytes_test, "stdlib", "str_bytes_test.zz"); parity_strict!(parity_stdlib_console, "stdlib", "console.zz"); parity_strict!(parity_stdlib_envmod, "stdlib", "envmod.zz"); parity_strict!(parity_stdlib_env_test, "stdlib", "env_test.zz"); diff --git a/crates/zz_cli/tests/e2e.rs b/crates/zz_cli/tests/e2e.rs index 50b8ddcd..79e042bb 100644 --- a/crates/zz_cli/tests/e2e.rs +++ b/crates/zz_cli/tests/e2e.rs @@ -613,6 +613,7 @@ e2e_success_test!( "str_extended_test.zz" ); e2e_success_test!(e2e_stdlib_str_find_test, "stdlib", "str_find_test.zz"); +e2e_success_test!(e2e_stdlib_str_bytes_test, "stdlib", "str_bytes_test.zz"); e2e_success_test!(e2e_stdlib_net_tcp_test, "stdlib", "net_tcp_test.zz"); e2e_success_test!(e2e_stdlib_input_chained, "stdlib", "input_chained.zz"); e2e_success_test!(e2e_stdlib_http_client_test, "stdlib", "http_client_test.zz"); diff --git a/crates/zz_codegen/src/lower/mod.rs b/crates/zz_codegen/src/lower/mod.rs index 21171d31..0bf36400 100644 --- a/crates/zz_codegen/src/lower/mod.rs +++ b/crates/zz_codegen/src/lower/mod.rs @@ -920,6 +920,9 @@ pub(crate) fn native_impl(name: &str) -> Option<&'static str> { "str.starts_with_at" | "std.str.starts_with_at" => Some("zz_str_starts_with_at"), "str.ends_with_at" | "std.str.ends_with_at" => Some("zz_str_ends_with_at"), "str.trim_span" | "std.str.trim_span" => Some("zz_str_trim_span"), + "str.bytes" | "std.str.bytes" => Some("zz_str_bytes"), + "bytes.to_str" | "std.bytes.to_str" => Some("zz_bytes_to_str"), + "bytes.to_ints" | "std.bytes.to_ints" => Some("zz_bytes_to_ints"), // math "math.abs" | "std.math.abs" => Some("zz_math_abs"), "math.sqrt" | "std.math.sqrt" => Some("zz_math_sqrt"), diff --git a/crates/zz_codegen/src/runtime/strings.c b/crates/zz_codegen/src/runtime/strings.c index ce1f1d4d..d1d92623 100644 --- a/crates/zz_codegen/src/runtime/strings.c +++ b/crates/zz_codegen/src/runtime/strings.c @@ -1554,6 +1554,96 @@ static size_t ws_width_bwd(const unsigned char *d, size_t s, size_t end) { return 0; } +// str.bytes(s) — UTF-8 bytes as plain ints (one copy). +zz_value zz_str_bytes(zz_value s, int *err) { + (void)err; + zz_value arr = zz_array_new(); + if (s.tag != ZZ_STR) return arr; + const unsigned char *d = (const unsigned char *)zz_str_ptr(s.s); + int sub_err = 0; + for (size_t i = 0; i < s.s->len; i++) { + zz_vec_append(arr, (zz_value){ZZ_INT, {.i = (int64_t)d[i]}}, &sub_err); + } + return arr; +} + +// bytes.to_str(vs) — strict UTF-8 decode; out-of-range values and +// invalid sequences are .err, identically on VM and AOT. +zz_value zz_bytes_to_str(zz_value vs, int *err) { + if (vs.tag != ZZ_ARRAY) { + if (err) *err = 1; + return zz_unit(); + } + size_t n = vs.arr->len; + unsigned char *buf = (unsigned char *)malloc(n > 0 ? n : 1); + if (!buf) { + return zz_variant_err(zz_str_static("bytes.to_str: out of memory")); + } + for (size_t i = 0; i < n; i++) { + zz_value v = vs.arr->items[i]; + if (v.tag != ZZ_INT) { + free(buf); + if (err) *err = 1; + return zz_unit(); + } + if (v.i < 0 || v.i > 255) { + free(buf); + char msg[96]; + snprintf(msg, sizeof(msg), "bytes.to_str: value %lld out of range 0-255", (long long)v.i); + return zz_variant_err(zz_str_new(msg, strlen(msg))); + } + buf[i] = (unsigned char)v.i; + } + // Strict validation: reject overlongs, surrogates, > U+10FFFF. + size_t i = 0; + int ok = 1; + while (i < n) { + unsigned char c = buf[i]; + size_t want = 1; + if (c < 0x80) want = 1; + else if (c >= 0xC2 && c <= 0xDF) want = 2; + else if (c >= 0xE0 && c <= 0xEF) want = 3; + else if (c >= 0xF0 && c <= 0xF4) want = 4; + else { ok = 0; break; } + if (i + want > n) { ok = 0; break; } + for (size_t k = 1; k < want; k++) { + if ((buf[i + k] & 0xC0) != 0x80) { ok = 0; break; } + } + if (!ok) break; + if (want == 3) { + if (c == 0xE0 && buf[i + 1] < 0xA0) { ok = 0; break; } + if (c == 0xED && buf[i + 1] > 0x9F) { ok = 0; break; } + } + if (want == 4) { + if (c == 0xF0 && buf[i + 1] < 0x90) { ok = 0; break; } + if (c == 0xF4 && buf[i + 1] > 0x8F) { ok = 0; break; } + } + i += want; + } + if (!ok) { + free(buf); + return zz_variant_err(zz_str_static("bytes.to_str: invalid UTF-8")); + } + zz_value out = zz_str_new((const char *)buf, n); + free(buf); + return zz_variant_ok(out); +} + +// bytes.to_ints(b) — opaque byte buffer as plain ints. +zz_value zz_bytes_to_ints(zz_value b, int *err) { + (void)err; + zz_value arr = zz_array_new(); + if (b.tag != ZZ_BYTES) return arr; + const unsigned char *d; + size_t n; + zz_bytes_view(b, &d, &n); + int sub_err = 0; + for (size_t i = 0; i < n; i++) { + zz_vec_append(arr, (zz_value){ZZ_INT, {.i = (int64_t)d[i]}}, &sub_err); + } + return arr; +} + // str.trim_span(s, start, end) — trimmed [lo, hi] byte offsets. Unicode // White_Space on both backends (explicit table, never the host trim). zz_value zz_str_trim_span(zz_value s, zz_value start, zz_value end, int *err) { diff --git a/crates/zz_codegen/src/runtime/strings.h b/crates/zz_codegen/src/runtime/strings.h index 11a0ad93..412b2b67 100644 --- a/crates/zz_codegen/src/runtime/strings.h +++ b/crates/zz_codegen/src/runtime/strings.h @@ -203,6 +203,9 @@ zz_value zz_str_find(zz_value s, zz_value sub, zz_value from, int *err); zz_value zz_str_rfind(zz_value s, zz_value sub, zz_value from, int *err); zz_value zz_str_starts_with_at(zz_value s, zz_value sub, zz_value pos, int *err); zz_value zz_str_ends_with_at(zz_value s, zz_value sub, zz_value pos, int *err); +zz_value zz_str_bytes(zz_value s, int *err); +zz_value zz_bytes_to_str(zz_value vs, int *err); +zz_value zz_bytes_to_ints(zz_value b, int *err); zz_value zz_str_trim_span(zz_value s, zz_value start, zz_value end, int *err); // ---- string casts ------------------------------------------------------ diff --git a/crates/zz_stdlib/src/funcs.rs b/crates/zz_stdlib/src/funcs.rs index 0263bfe9..a8bf8929 100644 --- a/crates/zz_stdlib/src/funcs.rs +++ b/crates/zz_stdlib/src/funcs.rs @@ -202,6 +202,10 @@ fn build_stdlib_funcs() -> HashMap { Type::Int, ), ); + m.insert( + "std.str.bytes".into(), + sig(vec![("s", Type::Str)], Type::Array(Box::new(Type::Int))), + ); m.insert( "std.str.starts_with_at".into(), sig( @@ -308,6 +312,10 @@ fn build_stdlib_funcs() -> HashMap { Type::Int, ), ); + m.insert( + "str.bytes".into(), + sig(vec![("s", Type::Str)], Type::Array(Box::new(Type::Int))), + ); m.insert( "str.starts_with_at".into(), sig( @@ -949,6 +957,28 @@ fn build_stdlib_funcs() -> HashMap { "std.bytes.len".into(), sig(vec![("v", Type::Bytes)], Type::Int), ); + m.insert( + "bytes.to_str".into(), + sig( + vec![("vs", Type::Array(Box::new(Type::Int)))], + Type::Result(Box::new(Type::Str), Box::new(Type::Str)), + ), + ); + m.insert( + "std.bytes.to_str".into(), + sig( + vec![("vs", Type::Array(Box::new(Type::Int)))], + Type::Result(Box::new(Type::Str), Box::new(Type::Str)), + ), + ); + m.insert( + "bytes.to_ints".into(), + sig(vec![("b", Type::Bytes)], Type::Array(Box::new(Type::Int))), + ); + m.insert( + "std.bytes.to_ints".into(), + sig(vec![("b", Type::Bytes)], Type::Array(Box::new(Type::Int))), + ); m.insert( "vec.push".into(), sig_t( @@ -4220,6 +4250,12 @@ mod tests { "str.ends_with_at", "std.str.trim_span", "str.trim_span", + "std.str.bytes", + "str.bytes", + "std.bytes.to_str", + "bytes.to_str", + "std.bytes.to_ints", + "bytes.to_ints", "std.str.find", "str.find", "std.str.rfind", @@ -4227,7 +4263,7 @@ mod tests { ] { assert!(funcs.contains_key(name), "missing {name}"); } - assert_eq!(funcs.len(), 888); + assert_eq!(funcs.len(), 894); } #[test] diff --git a/crates/zz_stdlib/src/natives/mod.rs b/crates/zz_stdlib/src/natives/mod.rs index b64adde8..170c67fb 100644 --- a/crates/zz_stdlib/src/natives/mod.rs +++ b/crates/zz_stdlib/src/natives/mod.rs @@ -168,6 +168,27 @@ fn build_stdlib_natives() -> HashMap { f: iterators::len, }, ); + m.insert( + "str.bytes".into(), + NativeEntry { + arity: 1, + f: str_mod::str_bytes, + }, + ); + m.insert( + "bytes.to_str".into(), + NativeEntry { + arity: 1, + f: str_mod::bytes_to_str, + }, + ); + m.insert( + "bytes.to_ints".into(), + NativeEntry { + arity: 1, + f: str_mod::bytes_to_ints, + }, + ); m.insert( "std.bytes.len".into(), NativeEntry { @@ -175,6 +196,27 @@ fn build_stdlib_natives() -> HashMap { f: iterators::len, }, ); + m.insert( + "std.str.bytes".into(), + NativeEntry { + arity: 1, + f: str_mod::str_bytes, + }, + ); + m.insert( + "std.bytes.to_str".into(), + NativeEntry { + arity: 1, + f: str_mod::bytes_to_str, + }, + ); + m.insert( + "std.bytes.to_ints".into(), + NativeEntry { + arity: 1, + f: str_mod::bytes_to_ints, + }, + ); m.insert( "map".into(), NativeEntry { diff --git a/crates/zz_stdlib/src/natives/str_mod/mod.rs b/crates/zz_stdlib/src/natives/str_mod/mod.rs index 7d694bbe..8f1ec710 100644 --- a/crates/zz_stdlib/src/natives/str_mod/mod.rs +++ b/crates/zz_stdlib/src/natives/str_mod/mod.rs @@ -1,4 +1,4 @@ -use crate::natives::{arg, expect_int, expect_str}; +use crate::natives::{arg, expect_array, expect_int, expect_str}; use zz_runtime::{EvalError, Interp, Span, Value}; pub(crate) fn str_length( @@ -108,6 +108,72 @@ fn byte_rfind(s: &str, sub: &str, from: i64) -> i64 { best.unwrap_or(-1) } +// str.bytes(s) — UTF-8 bytes as plain ints. One O(n) copy; the +// result composes with every [int] API (indexing, snapshots, the +// bytes.* builder vocabulary). The bridge find/rfind/trim_span need. +pub(crate) fn str_bytes( + _interp: &mut Interp, + args: &mut Vec, + _span: Span, +) -> Result { + let s = expect_str(args, 0, "std.str.bytes")?; + Ok(Value::Array(Box::new( + s.as_bytes().iter().map(|b| Value::Int(*b as i64)).collect(), + ))) +} + +// bytes.to_str(vs) — strict UTF-8 decode; invalid sequences and +// out-of-range values are .err (identical on VM and AOT). +pub(crate) fn bytes_to_str( + _interp: &mut Interp, + args: &mut Vec, + _span: Span, +) -> Result { + let vs = expect_array(args, 0, "bytes.to_str")?; + let mut buf = Vec::with_capacity(vs.len()); + for v in &vs { + match v { + Value::Int(n) => { + if !(0..=255).contains(n) { + return Ok(Value::Result(Box::new(Err(Value::Str(Box::new(format!( + "bytes.to_str: value {n} out of range 0-255" + ))))))); + } + buf.push(*n as u8); + } + other => { + return Err(EvalError::new( + format!("`bytes.to_str` expects an array of integers, found `{other}`"), + zz_runtime::Span::new(0, 0), + )); + } + } + } + match String::from_utf8(buf) { + Ok(s) => Ok(Value::Result(Box::new(Ok(Value::Str(Box::new(s)))))), + Err(_) => Ok(Value::Result(Box::new(Err(Value::Str(Box::new( + "bytes.to_str: invalid UTF-8".to_string(), + )))))), + } +} + +// bytes.to_ints(b) — opaque byte buffer as plain ints (zero-copy read). +pub(crate) fn bytes_to_ints( + _interp: &mut Interp, + args: &mut Vec, + _span: Span, +) -> Result { + match super::arg(args, 0, "bytes.to_ints")? { + Value::Bytes(b) => Ok(Value::Array(Box::new( + b.as_slice().iter().map(|x| Value::Int(*x as i64)).collect(), + ))), + other => Err(EvalError::new( + format!("`bytes.to_ints` expects bytes, found `{other}`"), + zz_runtime::Span::new(0, 0), + )), + } +} + pub(crate) fn str_starts_with_at( _interp: &mut Interp, args: &mut Vec, diff --git a/docs/stdlib.md b/docs/stdlib.md index 61a389d7..9a082f84 100644 --- a/docs/stdlib.md +++ b/docs/stdlib.md @@ -84,6 +84,9 @@ import std.str | `str.starts_with_at` | `str.starts_with_at(s: str, sub: str, pos: int) -> bool` | Match at byte offset (empty never matches) | | `str.ends_with_at` | `str.ends_with_at(s: str, sub: str, pos: int) -> bool` | Match ending at byte offset `pos` | | `str.trim_span` | `str.trim_span(s: str, start: int, end: int) -> [int]` | Trimmed `[lo, hi]` byte offsets (Unicode ws, both backends) | +| `str.bytes` | `str.bytes(s: str) -> [int]` | UTF-8 bytes as plain ints (one copy) | +| `bytes.to_str` | `bytes.to_str(vs: [int]) -> Result` | Strict UTF-8 decode; range/invalid input is `.err` on both backends | +| `bytes.to_ints` | `bytes.to_ints(b: bytes) -> [int]` | Opaque byte buffer as plain ints | Offsets are bytes (O(1) per call, O(n) streaming total; matches Rust `str::find` semantics). `length`/slicing stay char-oriented — convert @@ -108,6 +111,9 @@ str.rfind("hello world", "o", 10) // 7 str.starts_with_at("hello", "ell", 1) // true str.ends_with_at("hello", "ell", 4) // true str.trim_span(" hi ", 0, 6) // [2, 4] +str.bytes("AB") // [65, 66] +bytes.to_str([104, 105]) // .ok("hi") +bytes.to_ints(b)[0] // first byte as int ``` ### Method Call Syntax diff --git a/tests/fixtures/stdlib/str_bytes_test.zz b/tests/fixtures/stdlib/str_bytes_test.zz new file mode 100644 index 00000000..9d80123e --- /dev/null +++ b/tests/fixtures/stdlib/str_bytes_test.zz @@ -0,0 +1,52 @@ +// features: imports, locals, match, print, std-str, std-bytes, string-literal +// E2E: str.bytes / bytes.to_str / bytes.to_ints roundtrips + error paths. +import std.str +import std.bytes +import std.encoding + +func main() { + b := str.bytes("ABC") + println("len={vec.len(b)}") + println("b0={b[0]},b2={b[2]}") + uni := str.bytes("é") + println("unilen={vec.len(uni)}") + println("uni0={uni[0]},uni1={uni[1]}") + match bytes.to_str(b) { + .ok(s) => println("roundtrip={s}"), + .err(_) => println("roundtrip_bad"), + } + match bytes.to_str(uni) { + .ok(s) => println("uniroundtrip={s}"), + .err(_) => println("uniroundtrip_bad"), + } + match bytes.to_str([104, 105]) { + .ok(s) => println("lit={s}"), + .err(_) => println("lit_bad"), + } + match bytes.to_str([300]) { + .ok(_) => println("range_bad"), + .err(e) => println("range_err={e}"), + } + match bytes.to_str([255, 254]) { + .ok(_) => println("invalid_bad"), + .err(e) => println("invalid_err={e}"), + } + match bytes.to_str([]) { + .ok(s) => { + empty_ok := s == "" + println("empty_ok={empty_ok}") + }, + .err(_) => println("empty_bad"), + } + ab := std.str.bytes("AB") + println("stdspell={ab[1]}") + match encoding.base64_decode_bytes("aGk=") { + .ok(rb) => { + ints := bytes.to_ints(rb) + println("intslen={vec.len(ints)}") + println("ints0={ints[0]},ints1={ints[1]}") + }, + .err(_) => println("b64_bad"), + } + println("str_bytes_ok") +}