Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/zz_cli/tests/dual_engine_parity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1136,6 +1136,7 @@ parity_strict!(
"vec_nested_str_parity.zz"
);
parity_strict!(parity_stdlib_str_find_test, "stdlib", "str_find_test.zz");
parity_strict!(parity_stdlib_str_bytes_test, "stdlib", "str_bytes_test.zz");
parity_strict!(parity_stdlib_console, "stdlib", "console.zz");
parity_strict!(parity_stdlib_envmod, "stdlib", "envmod.zz");
parity_strict!(parity_stdlib_env_test, "stdlib", "env_test.zz");
Expand Down
1 change: 1 addition & 0 deletions crates/zz_cli/tests/e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,7 @@ e2e_success_test!(
"str_extended_test.zz"
);
e2e_success_test!(e2e_stdlib_str_find_test, "stdlib", "str_find_test.zz");
e2e_success_test!(e2e_stdlib_str_bytes_test, "stdlib", "str_bytes_test.zz");
e2e_success_test!(e2e_stdlib_net_tcp_test, "stdlib", "net_tcp_test.zz");
e2e_success_test!(e2e_stdlib_input_chained, "stdlib", "input_chained.zz");
e2e_success_test!(e2e_stdlib_http_client_test, "stdlib", "http_client_test.zz");
Expand Down
3 changes: 3 additions & 0 deletions crates/zz_codegen/src/lower/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,9 @@ pub(crate) fn native_impl(name: &str) -> Option<&'static str> {
"str.starts_with_at" | "std.str.starts_with_at" => Some("zz_str_starts_with_at"),
"str.ends_with_at" | "std.str.ends_with_at" => Some("zz_str_ends_with_at"),
"str.trim_span" | "std.str.trim_span" => Some("zz_str_trim_span"),
"str.bytes" | "std.str.bytes" => Some("zz_str_bytes"),
"bytes.to_str" | "std.bytes.to_str" => Some("zz_bytes_to_str"),
"bytes.to_ints" | "std.bytes.to_ints" => Some("zz_bytes_to_ints"),
// math
"math.abs" | "std.math.abs" => Some("zz_math_abs"),
"math.sqrt" | "std.math.sqrt" => Some("zz_math_sqrt"),
Expand Down
90 changes: 90 additions & 0 deletions crates/zz_codegen/src/runtime/strings.c
Original file line number Diff line number Diff line change
Expand Up @@ -1554,6 +1554,96 @@ static size_t ws_width_bwd(const unsigned char *d, size_t s, size_t end) {
return 0;
}

// str.bytes(s) — UTF-8 bytes as plain ints (one copy).
zz_value zz_str_bytes(zz_value s, int *err) {
(void)err;
zz_value arr = zz_array_new();
if (s.tag != ZZ_STR) return arr;
const unsigned char *d = (const unsigned char *)zz_str_ptr(s.s);
int sub_err = 0;
for (size_t i = 0; i < s.s->len; i++) {
zz_vec_append(arr, (zz_value){ZZ_INT, {.i = (int64_t)d[i]}}, &sub_err);
}
return arr;
}

// bytes.to_str(vs) — strict UTF-8 decode; out-of-range values and
// invalid sequences are .err, identically on VM and AOT.
zz_value zz_bytes_to_str(zz_value vs, int *err) {
if (vs.tag != ZZ_ARRAY) {
if (err) *err = 1;
return zz_unit();
}
size_t n = vs.arr->len;
unsigned char *buf = (unsigned char *)malloc(n > 0 ? n : 1);
if (!buf) {
return zz_variant_err(zz_str_static("bytes.to_str: out of memory"));
}
for (size_t i = 0; i < n; i++) {
zz_value v = vs.arr->items[i];
if (v.tag != ZZ_INT) {
free(buf);
if (err) *err = 1;
return zz_unit();
}
if (v.i < 0 || v.i > 255) {
free(buf);
char msg[96];
snprintf(msg, sizeof(msg), "bytes.to_str: value %lld out of range 0-255", (long long)v.i);
return zz_variant_err(zz_str_new(msg, strlen(msg)));
}
buf[i] = (unsigned char)v.i;
}
// Strict validation: reject overlongs, surrogates, > U+10FFFF.
size_t i = 0;
int ok = 1;
while (i < n) {
unsigned char c = buf[i];
size_t want = 1;
if (c < 0x80) want = 1;
else if (c >= 0xC2 && c <= 0xDF) want = 2;
else if (c >= 0xE0 && c <= 0xEF) want = 3;
else if (c >= 0xF0 && c <= 0xF4) want = 4;
else { ok = 0; break; }
if (i + want > n) { ok = 0; break; }
for (size_t k = 1; k < want; k++) {
if ((buf[i + k] & 0xC0) != 0x80) { ok = 0; break; }
}
if (!ok) break;
if (want == 3) {
if (c == 0xE0 && buf[i + 1] < 0xA0) { ok = 0; break; }
if (c == 0xED && buf[i + 1] > 0x9F) { ok = 0; break; }
}
if (want == 4) {
if (c == 0xF0 && buf[i + 1] < 0x90) { ok = 0; break; }
if (c == 0xF4 && buf[i + 1] > 0x8F) { ok = 0; break; }
}
i += want;
}
if (!ok) {
free(buf);
return zz_variant_err(zz_str_static("bytes.to_str: invalid UTF-8"));
}
zz_value out = zz_str_new((const char *)buf, n);
free(buf);
return zz_variant_ok(out);
}

// bytes.to_ints(b) — opaque byte buffer as plain ints.
zz_value zz_bytes_to_ints(zz_value b, int *err) {
(void)err;
zz_value arr = zz_array_new();
if (b.tag != ZZ_BYTES) return arr;
const unsigned char *d;
size_t n;
zz_bytes_view(b, &d, &n);
int sub_err = 0;
for (size_t i = 0; i < n; i++) {
zz_vec_append(arr, (zz_value){ZZ_INT, {.i = (int64_t)d[i]}}, &sub_err);
}
return arr;
}

// str.trim_span(s, start, end) — trimmed [lo, hi] byte offsets. Unicode
// White_Space on both backends (explicit table, never the host trim).
zz_value zz_str_trim_span(zz_value s, zz_value start, zz_value end, int *err) {
Expand Down
3 changes: 3 additions & 0 deletions crates/zz_codegen/src/runtime/strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,9 @@ zz_value zz_str_find(zz_value s, zz_value sub, zz_value from, int *err);
zz_value zz_str_rfind(zz_value s, zz_value sub, zz_value from, int *err);
zz_value zz_str_starts_with_at(zz_value s, zz_value sub, zz_value pos, int *err);
zz_value zz_str_ends_with_at(zz_value s, zz_value sub, zz_value pos, int *err);
zz_value zz_str_bytes(zz_value s, int *err);
zz_value zz_bytes_to_str(zz_value vs, int *err);
zz_value zz_bytes_to_ints(zz_value b, int *err);
zz_value zz_str_trim_span(zz_value s, zz_value start, zz_value end, int *err);

// ---- string casts ------------------------------------------------------
Expand Down
38 changes: 37 additions & 1 deletion crates/zz_stdlib/src/funcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ fn build_stdlib_funcs() -> HashMap<String, FuncSig> {
Type::Int,
),
);
m.insert(
"std.str.bytes".into(),
sig(vec![("s", Type::Str)], Type::Array(Box::new(Type::Int))),
);
m.insert(
"std.str.starts_with_at".into(),
sig(
Expand Down Expand Up @@ -308,6 +312,10 @@ fn build_stdlib_funcs() -> HashMap<String, FuncSig> {
Type::Int,
),
);
m.insert(
"str.bytes".into(),
sig(vec![("s", Type::Str)], Type::Array(Box::new(Type::Int))),
);
m.insert(
"str.starts_with_at".into(),
sig(
Expand Down Expand Up @@ -949,6 +957,28 @@ fn build_stdlib_funcs() -> HashMap<String, FuncSig> {
"std.bytes.len".into(),
sig(vec![("v", Type::Bytes)], Type::Int),
);
m.insert(
"bytes.to_str".into(),
sig(
vec![("vs", Type::Array(Box::new(Type::Int)))],
Type::Result(Box::new(Type::Str), Box::new(Type::Str)),
),
);
m.insert(
"std.bytes.to_str".into(),
sig(
vec![("vs", Type::Array(Box::new(Type::Int)))],
Type::Result(Box::new(Type::Str), Box::new(Type::Str)),
),
);
m.insert(
"bytes.to_ints".into(),
sig(vec![("b", Type::Bytes)], Type::Array(Box::new(Type::Int))),
);
m.insert(
"std.bytes.to_ints".into(),
sig(vec![("b", Type::Bytes)], Type::Array(Box::new(Type::Int))),
);
m.insert(
"vec.push".into(),
sig_t(
Expand Down Expand Up @@ -4220,14 +4250,20 @@ mod tests {
"str.ends_with_at",
"std.str.trim_span",
"str.trim_span",
"std.str.bytes",
"str.bytes",
"std.bytes.to_str",
"bytes.to_str",
"std.bytes.to_ints",
"bytes.to_ints",
"std.str.find",
"str.find",
"std.str.rfind",
"str.rfind",
] {
assert!(funcs.contains_key(name), "missing {name}");
}
assert_eq!(funcs.len(), 888);
assert_eq!(funcs.len(), 894);
}

#[test]
Expand Down
42 changes: 42 additions & 0 deletions crates/zz_stdlib/src/natives/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,13 +168,55 @@ fn build_stdlib_natives() -> HashMap<String, NativeEntry> {
f: iterators::len,
},
);
m.insert(
"str.bytes".into(),
NativeEntry {
arity: 1,
f: str_mod::str_bytes,
},
);
m.insert(
"bytes.to_str".into(),
NativeEntry {
arity: 1,
f: str_mod::bytes_to_str,
},
);
m.insert(
"bytes.to_ints".into(),
NativeEntry {
arity: 1,
f: str_mod::bytes_to_ints,
},
);
m.insert(
"std.bytes.len".into(),
NativeEntry {
arity: 1,
f: iterators::len,
},
);
m.insert(
"std.str.bytes".into(),
NativeEntry {
arity: 1,
f: str_mod::str_bytes,
},
);
m.insert(
"std.bytes.to_str".into(),
NativeEntry {
arity: 1,
f: str_mod::bytes_to_str,
},
);
m.insert(
"std.bytes.to_ints".into(),
NativeEntry {
arity: 1,
f: str_mod::bytes_to_ints,
},
);
m.insert(
"map".into(),
NativeEntry {
Expand Down
68 changes: 67 additions & 1 deletion crates/zz_stdlib/src/natives/str_mod/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::natives::{arg, expect_int, expect_str};
use crate::natives::{arg, expect_array, expect_int, expect_str};
use zz_runtime::{EvalError, Interp, Span, Value};

pub(crate) fn str_length(
Expand Down Expand Up @@ -108,6 +108,72 @@ fn byte_rfind(s: &str, sub: &str, from: i64) -> i64 {
best.unwrap_or(-1)
}

// str.bytes(s) — UTF-8 bytes as plain ints. One O(n) copy; the
// result composes with every [int] API (indexing, snapshots, the
// bytes.* builder vocabulary). The bridge find/rfind/trim_span need.
pub(crate) fn str_bytes(
_interp: &mut Interp,
args: &mut Vec<Value>,
_span: Span,
) -> Result<Value, EvalError> {
let s = expect_str(args, 0, "std.str.bytes")?;
Ok(Value::Array(Box::new(
s.as_bytes().iter().map(|b| Value::Int(*b as i64)).collect(),
)))
}

// bytes.to_str(vs) — strict UTF-8 decode; invalid sequences and
// out-of-range values are .err (identical on VM and AOT).
pub(crate) fn bytes_to_str(
_interp: &mut Interp,
args: &mut Vec<Value>,
_span: Span,
) -> Result<Value, EvalError> {
let vs = expect_array(args, 0, "bytes.to_str")?;
let mut buf = Vec::with_capacity(vs.len());
for v in &vs {
match v {
Value::Int(n) => {
if !(0..=255).contains(n) {
return Ok(Value::Result(Box::new(Err(Value::Str(Box::new(format!(
"bytes.to_str: value {n} out of range 0-255"
)))))));
}
buf.push(*n as u8);
}
other => {
return Err(EvalError::new(
format!("`bytes.to_str` expects an array of integers, found `{other}`"),
zz_runtime::Span::new(0, 0),
));
}
}
}
match String::from_utf8(buf) {
Ok(s) => Ok(Value::Result(Box::new(Ok(Value::Str(Box::new(s)))))),
Err(_) => Ok(Value::Result(Box::new(Err(Value::Str(Box::new(
"bytes.to_str: invalid UTF-8".to_string(),
)))))),
}
}

// bytes.to_ints(b) — opaque byte buffer as plain ints (zero-copy read).
pub(crate) fn bytes_to_ints(
_interp: &mut Interp,
args: &mut Vec<Value>,
_span: Span,
) -> Result<Value, EvalError> {
match super::arg(args, 0, "bytes.to_ints")? {
Value::Bytes(b) => Ok(Value::Array(Box::new(
b.as_slice().iter().map(|x| Value::Int(*x as i64)).collect(),
))),
other => Err(EvalError::new(
format!("`bytes.to_ints` expects bytes, found `{other}`"),
zz_runtime::Span::new(0, 0),
)),
}
}

pub(crate) fn str_starts_with_at(
_interp: &mut Interp,
args: &mut Vec<Value>,
Expand Down
6 changes: 6 additions & 0 deletions docs/stdlib.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ import std.str
| `str.starts_with_at` | `str.starts_with_at(s: str, sub: str, pos: int) -> bool` | Match at byte offset (empty never matches) |
| `str.ends_with_at` | `str.ends_with_at(s: str, sub: str, pos: int) -> bool` | Match ending at byte offset `pos` |
| `str.trim_span` | `str.trim_span(s: str, start: int, end: int) -> [int]` | Trimmed `[lo, hi]` byte offsets (Unicode ws, both backends) |
| `str.bytes` | `str.bytes(s: str) -> [int]` | UTF-8 bytes as plain ints (one copy) |
| `bytes.to_str` | `bytes.to_str(vs: [int]) -> Result<str>` | Strict UTF-8 decode; range/invalid input is `.err` on both backends |
| `bytes.to_ints` | `bytes.to_ints(b: bytes) -> [int]` | Opaque byte buffer as plain ints |

Offsets are bytes (O(1) per call, O(n) streaming total; matches Rust
`str::find` semantics). `length`/slicing stay char-oriented — convert
Expand All @@ -108,6 +111,9 @@ str.rfind("hello world", "o", 10) // 7
str.starts_with_at("hello", "ell", 1) // true
str.ends_with_at("hello", "ell", 4) // true
str.trim_span(" hi ", 0, 6) // [2, 4]
str.bytes("AB") // [65, 66]
bytes.to_str([104, 105]) // .ok("hi")
bytes.to_ints(b)[0] // first byte as int
```

### Method Call Syntax
Expand Down
Loading
Loading