diff --git a/CHANGELOG.md b/CHANGELOG.md index dc07e3b..1efa34f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,12 +40,26 @@ release notes. ### Diagnostics +- `deed test --compiled` runs the properties contracts generate, rather than + stopping after written test blocks. The corpus now reports 118 compiled + passes instead of 111: the same seven generated properties the interpreter + runs, alongside the blocks the backend can compile. The remaining difference + is written code the backend names in its six existing skip records. + + Generated inputs use the same generator and precondition filtering as the + interpreter. Each accepted value is written into the compiled module's own + memory using the MIR layout, the export is called, and its answer is decoded + from that memory and compared structurally with the reference answer. The + interpreter remains the one implementation of contract evaluation, so this + does not create a second `ensures` evaluator that could drift from the first. + A wrong contract still fails the property, and a backend answer that differs + from the interpreter fails it independently. + - `deed test --compiled` says what it did not compile. It ran a hundred and - eleven of the corpus's blocks where the interpreter ran a hundred and - thirty-eight, and reported `111 passed, 0 failed`, which is what a complete - run looks like. A file the backend refused outright said `no tests found in - the compiled backend` and exited zero, which is what a file with no tests in - it says. + eleven results where the interpreter reported a hundred and thirty-eight, + and printed `111 passed, 0 failed`, which is what a complete run looks like. + A file the backend refused outright said `no tests found in the compiled + backend` and exited zero, which is what a file with no tests in it says. Skipping is right — the backend compiles a subset of the language on purpose — and staying quiet about it was not. Every block that is dropped is now diff --git a/crates/deed-cli/src/args.rs b/crates/deed-cli/src/args.rs index ff4cc18..b440790 100644 --- a/crates/deed-cli/src/args.rs +++ b/crates/deed-cli/src/args.rs @@ -42,8 +42,8 @@ Options: than all of it. --check With `fmt` or `fix`, change nothing and report what would have changed. - --compiled With `test`, run test blocks through the compiled - WebAssembly backend instead of the interpreter. + --compiled With `test`, run test blocks and generated properties + through the compiled WebAssembly backend. --component With `build`, write a core module, the `.wit` world its exports describe, and a component binary, instead of a standalone program. The component is written @@ -72,9 +72,9 @@ the directory and the module path, so it is lowercase letters, digits and `_`. It writes no manifest: a manifest here says where code outside your tree lives, and a new project has none. `deed test` refuses to run anything that does not check. -`deed test --compiled` runs the same test blocks through the compiled backend. - Blocks the backend cannot compile are skipped, and the count of the ones that - ran has to match what the interpreter ran. +`deed test --compiled` runs test blocks and the properties contracts generate +through the compiled backend. Blocks the backend cannot compile are skipped and +named, so the summary says both how much ran and what did not. `deed run` calls `main`, handing it the one `System` capability there is. Everything after `--` goes to the program, which reads it with `Io.args`. Standard input is read when, and only when, `main`'s row says `Io.line`. A diff --git a/crates/deed-cli/src/main.rs b/crates/deed-cli/src/main.rs index e7100fe..780b13c 100644 --- a/crates/deed-cli/src/main.rs +++ b/crates/deed-cli/src/main.rs @@ -13,7 +13,7 @@ use std::process::ExitCode; use deed_ast::Item; use deed_diagnostics::{Diagnostic, FileId, SourceMap, render_human}; use deed_driver::{Checked, ObligationReport}; -use deed_interp::{PropertyConfig, RuntimeProfile}; +use deed_interp::{PropertyAttempt, PropertyConfig, PropertyInterpreter, RuntimeProfile}; use deed_typeck::Tier; use crate::args::{CheckArgs, Command, Format, Mode, USAGE}; @@ -1087,6 +1087,303 @@ fn compiled_args(module: &deed_codegen::Module, name: &str) -> Option, +) -> Option { + match (ty, value) { + (deed_mir::Ty::Unit, deed_interp::Value::Unit) => Some(deed_codegen::Value::I64(0)), + (deed_mir::Ty::Bool, deed_interp::Value::Bool(value)) => { + Some(deed_codegen::Value::I32(i32::from(*value))) + } + (deed_mir::Ty::Int, deed_interp::Value::Int(value)) => { + Some(deed_codegen::Value::I64(*value)) + } + (deed_mir::Ty::Str, deed_interp::Value::Str(value)) => memory.write_text(value), + (deed_mir::Ty::List(element), deed_interp::Value::List(values)) => { + let values = values + .iter() + .map(|value| encode_property_value(program, element, value, memory)) + .collect::>>()?; + memory.write_list(&values) + } + (deed_mir::Ty::Aggregate(id), value) => { + let layout = program.layouts.get(id.0)?; + let variant_name = match value { + deed_interp::Value::Record(_) => layout.variants.first()?.name.as_str(), + deed_interp::Value::Variant(value) => value.name.as_str(), + deed_interp::Value::Result { ok, .. } => { + if *ok { + "ok" + } else { + "err" + } + } + _ => return None, + }; + let variant_index = layout + .variants + .iter() + .position(|variant| variant.name == variant_name)?; + let variant = &layout.variants[variant_index]; + let mut fields = Vec::with_capacity(variant.fields.len()); + for field in &variant.fields { + let value = match value { + deed_interp::Value::Record(values) => values.get(&field.name)?, + deed_interp::Value::Variant(value) => value.fields.get(&field.name)?, + deed_interp::Value::Result { value, .. } => value, + _ => return None, + }; + fields.push(encode_property_value(program, &field.ty, value, memory)?); + } + memory.write_aggregate(layout.is_tagged().then_some(variant_index as i64), &fields) + } + _ => None, + } +} + +fn encode_property_args( + program: &deed_mir::Program, + function: &str, + values: &[deed_interp::Value], + memory: &mut deed_codegen::HostCall<'_>, +) -> Option> { + let function = program.function(program.find(function)?); + if function.params.len() != values.len() { + return None; + } + + let mut args = Vec::new(); + for (ty, value) in function.params.iter().zip(values) { + if matches!(ty, deed_mir::Ty::Unit) { + if !matches!(value, deed_interp::Value::Unit) { + return None; + } + continue; + } + args.push(encode_property_value(program, ty, value, memory)?); + } + Some(args) +} + +fn property_word(memory: &[u8], at: usize) -> Option { + let bytes = memory.get(at..at.checked_add(deed_codegen::layout::WORD as usize)?)?; + Some(i64::from_le_bytes(bytes.try_into().ok()?)) +} + +fn decode_property_word( + program: &deed_mir::Program, + ty: &deed_mir::Ty, + word: i64, + memory: &[u8], + expected: Option<&deed_interp::Value>, + remaining: &[()], +) -> Option { + let (_, remaining) = remaining.split_first()?; + match ty { + deed_mir::Ty::Unit => Some(deed_interp::Value::Unit), + deed_mir::Ty::Bool => Some(deed_interp::Value::Bool(word != 0)), + deed_mir::Ty::Int => Some(deed_interp::Value::Int(word)), + deed_mir::Ty::Str => { + let at = usize::try_from(word).ok()?; + let characters = usize::try_from(property_word(memory, at)?).ok()?; + let bytes = usize::try_from(property_word( + memory, + at.checked_add(deed_codegen::layout::WORD as usize)?, + )?) + .ok()?; + let start = at.checked_add(2 * deed_codegen::layout::WORD as usize)?; + let text = std::str::from_utf8(memory.get(start..start.checked_add(bytes)?)?).ok()?; + (text.chars().count() == characters).then(|| deed_interp::Value::str(text)) + } + deed_mir::Ty::List(element) => { + let at = usize::try_from(word).ok()?; + let len = usize::try_from(property_word(memory, at)?).ok()?; + let elements = at.checked_add(deed_codegen::layout::WORD as usize)?; + let bytes = len.checked_mul(deed_codegen::layout::WORD as usize)?; + memory.get(elements..elements.checked_add(bytes)?)?; + let expected = match expected { + Some(deed_interp::Value::List(values)) => Some(values.as_slice()), + _ => None, + }; + let mut values = Vec::with_capacity(len); + for index in 0..len { + let slot = at.checked_add(deed_codegen::layout::element_offset(index) as usize)?; + values.push(decode_property_word( + program, + element, + property_word(memory, slot)?, + memory, + expected.and_then(|values| values.get(index)), + remaining, + )?); + } + Some(deed_interp::Value::List(std::rc::Rc::new(values))) + } + deed_mir::Ty::Aggregate(id) => { + let layout = program.layouts.get(id.0)?; + let at = usize::try_from(word).ok()?; + let variant_index = if layout.is_tagged() { + usize::try_from(property_word(memory, at)?).ok()? + } else { + 0 + }; + let variant = layout.variants.get(variant_index)?; + let expected_fields = match expected { + Some(deed_interp::Value::Record(fields)) => Some(fields.as_ref()), + Some(deed_interp::Value::Variant(value)) => Some(&value.fields), + _ => None, + }; + let expected_result = match expected { + Some(deed_interp::Value::Result { value, .. }) => Some(value.as_ref()), + _ => None, + }; + let mut fields = deed_interp::Fields::new(); + for (index, field) in variant.fields.iter().enumerate() { + let slot = at + .checked_add( + deed_codegen::layout::field_offset(layout.is_tagged(), index) as usize, + )?; + let expected = expected_fields + .and_then(|fields| fields.get(&field.name)) + .or(expected_result); + fields.insert( + field.name.clone(), + decode_property_word( + program, + &field.ty, + property_word(memory, slot)?, + memory, + expected, + remaining, + )?, + ); + } + + if layout.name.starts_with("Result<") { + let payload = fields.remove("value")?; + return match variant.name.as_str() { + "ok" => Some(deed_interp::Value::ok(payload)), + "err" => Some(deed_interp::Value::err(payload)), + _ => None, + }; + } + if layout.choice { + let origin = match expected? { + deed_interp::Value::Variant(value) => std::rc::Rc::clone(&value.origin), + _ => return None, + }; + Some(deed_interp::Value::variant( + origin, + variant.name.clone(), + fields, + )) + } else { + Some(deed_interp::Value::record(fields)) + } + } + deed_mir::Ty::Capability | deed_mir::Ty::Closure => None, + } +} + +fn decode_property_result( + program: &deed_mir::Program, + function: &str, + compiled: Option, + memory: &[u8], + expected: &deed_interp::Value, +) -> Option { + let ty = &program.function(program.find(function)?).ret; + match (ty, compiled) { + (deed_mir::Ty::Unit, None) => Some(deed_interp::Value::Unit), + (deed_mir::Ty::Bool, Some(deed_codegen::Value::I32(value))) => { + Some(deed_interp::Value::Bool(value != 0)) + } + (ty, Some(deed_codegen::Value::I64(word))) => { + decode_property_word(program, ty, word, memory, Some(expected), &[(); 65]) + } + _ => None, + } +} + +fn compiled_property_attempt<'a>( + file: FileId, + lowered: &deed_mir::Program, + compiled: &deed_codegen::Module, + reference: &mut PropertyInterpreter<'a>, + function: &'a deed_ast::FnDecl, + values: &[deed_interp::Value], +) -> PropertyAttempt { + let interpreted = reference.attempt(function, values); + if matches!(interpreted, PropertyAttempt::Rejected) { + return PropertyAttempt::Rejected; + } + + let compiled_outcome = deed_codegen::call_prepared(compiled, &function.sig.name.name, |call| { + encode_property_args(lowered, &function.sig.name.name, values, call).ok_or_else(|| { + deed_codegen::Trap::Unimplemented(format!( + "the generated inputs to `{}` do not cross the compiled call boundary", + function.sig.name.name + )) + }) + }); + match interpreted { + PropertyAttempt::Failed(diagnostic) => PropertyAttempt::Failed(diagnostic), + PropertyAttempt::Passed(value) => match compiled_outcome { + Ok((answer, memory)) => match decode_property_result( + lowered, + &function.sig.name.name, + answer, + &memory, + &value, + ) { + Some(decoded) if decoded == value => PropertyAttempt::Passed(value), + Some(decoded) => PropertyAttempt::Failed( + Diagnostic::error( + deed_mir::codes::ASSERTION_FAILED, + file, + function.sig.name.span, + format!( + "the compiled `{}` answered {decoded:?}, while the interpreter answered {value:?}", + function.sig.name.name + ), + ) + .with_primary_label("the two runtimes disagreed on this generated input"), + ), + None => PropertyAttempt::Failed( + Diagnostic::error( + deed_mir::codes::NOT_RUNNABLE, + file, + function.sig.name.span, + format!( + "the compiled property runner could not read `{}`'s answer", + function.sig.name.name + ), + ) + .with_primary_label("this compiled answer could not be decoded"), + ), + }, + Err(trap) => PropertyAttempt::Failed( + compiled_diagnostic(file, &trap).unwrap_or_else(|| { + Diagnostic::error( + deed_mir::codes::NOT_RUNNABLE, + file, + function.sig.name.span, + format!( + "the compiled `{}` stopped on a generated input: {trap}", + function.sig.name.name + ), + ) + .with_primary_label("the compiled property run stopped here") + }), + ), + }, + PropertyAttempt::Rejected => unreachable!("rejected inputs returned above"), + } +} + fn run_compiled_main( out: &mut impl Write, sources: &SourceMap, @@ -1624,6 +1921,7 @@ fn run_compiled_tests( let mut failed: Vec<(String, String)> = Vec::new(); let mut ran = 0usize; let mut skipped: Vec = Vec::new(); + let program = deed_driver::program_of(checks); for (at, checked) in checks[..subject.min(checks.len())].iter().enumerate() { let name = sources.file(checked.file).name().to_string(); @@ -1644,7 +1942,10 @@ fn run_compiled_tests( skipped.push(format!("{name}: test {:?}: {}", block.name, block.why)); } - if lowered.tests.is_empty() { + let has_properties = checked.module.items.iter().any(|item| { + matches!(item, deed_ast::Item::Function(function) if deed_interp::is_testable(function, &checked.module, &checked.resolutions)) + }); + if lowered.tests.is_empty() && !has_properties { continue; } @@ -1702,6 +2003,39 @@ fn run_compiled_tests( writeln!(out, " FAIL {label}")?; } } + + let mut reference = PropertyInterpreter::new(&program, checked.file); + let properties = deed_interp::run_properties_with( + &program, + checked.file, + &checked.module, + &checked.resolutions, + PropertyConfig::default(), + |function, values| { + compiled_property_attempt( + checked.file, + &lowered, + &compiled, + &mut reference, + function, + values, + ) + }, + ); + for property in properties { + ran += 1; + let label = format!("property {} ({} cases)", property.function, property.cases); + match property.failure { + None => { + passed += 1; + writeln!(out, " ok {label}")?; + } + Some(diagnostic) => { + writeln!(out, " FAIL {label}")?; + failed.push((label, render_human(sources, &diagnostic))); + } + } + } } if ran == 0 { @@ -2757,4 +3091,169 @@ mod compiled_tests { ); assert!(compiled_args(&module, "missing").is_none()); } + + #[test] + fn a_compiled_property_uses_the_compiled_answer() { + let mut sources = SourceMap::new(); + let checked = deed_driver::check_text( + &mut sources, + "test.deed", + "module a\n\n\ + fn same(n: Int) -> Int\n\ + ensures ok => result == n,\n\ + { n }\n", + ); + let function = checked + .module + .items + .iter() + .find_map(|item| match item { + deed_ast::Item::Function(function) => Some(function), + _ => None, + }) + .expect("the function should parse"); + + let mut program = deed_interp::Program::new(); + program.add( + checked.file, + &checked.module, + &checked.resolutions, + checked.guards(), + checked.rows(), + checked.operators(), + ); + let mut reference = PropertyInterpreter::new(&program, checked.file); + + let mut compiled = deed_codegen::Module::new(); + let ty = compiled.intern_type(FuncType { + params: vec![ValType::I64], + results: vec![ValType::I64], + }); + let function_index = compiled.add_func(Func { + type_index: ty, + locals: vec![], + body: vec![Ins::I64Const(0), Ins::Return], + }); + compiled.export("same", function_index); + + let outcome = compiled_property_attempt( + checked.file, + &{ + let mut program = deed_mir::Program::new(); + program.add_function(deed_mir::Function::new( + "same", + vec![deed_mir::Ty::Int], + deed_mir::Ty::Int, + )); + program + }, + &compiled, + &mut reference, + function, + &[deed_interp::Value::Int(7)], + ); + let PropertyAttempt::Failed(diagnostic) = outcome else { + panic!("the deliberately different compiled answer should fail"); + }; + assert_eq!(diagnostic.code, deed_mir::codes::ASSERTION_FAILED); + } + + #[test] + fn a_compiled_property_reads_the_boxed_answer_from_memory() { + let mut sources = SourceMap::new(); + let checked = deed_driver::check_text( + &mut sources, + "test.deed", + "module a\n\n\ + fn maybe_one() -> Result\n\ + ensures ok => result == 1,\n\ + { ok(1) }\n", + ); + let function = checked + .module + .items + .iter() + .find_map(|item| match item { + deed_ast::Item::Function(function) => Some(function), + _ => None, + }) + .expect("the function should parse"); + + let mut program = deed_interp::Program::new(); + program.add( + checked.file, + &checked.module, + &checked.resolutions, + checked.guards(), + checked.rows(), + checked.operators(), + ); + let mut reference = PropertyInterpreter::new(&program, checked.file); + + let lowered = deed_mir::lower(&checked.module, &checked.resolutions, &checked.types) + .expect("the function should lower"); + let mut compiled = deed_codegen::compile(&lowered).expect("the function should compile"); + let exported = compiled + .exports + .iter() + .find(|(name, _)| name == "maybe_one") + .map(|(_, index)| *index) + .expect("the function should be exported"); + let function_index = exported as usize - compiled.imports.len(); + let answer = deed_codegen::layout::HEAP_START + 1024; + compiled.funcs[function_index].body = vec![Ins::I64Const(answer as i64), Ins::Return]; + let mut aggregate = Vec::new(); + aggregate.extend_from_slice(&0i64.to_le_bytes()); + aggregate.extend_from_slice(&2i64.to_le_bytes()); + compiled.data.push((answer, aggregate)); + + let outcome = compiled_property_attempt( + checked.file, + &lowered, + &compiled, + &mut reference, + function, + &[], + ); + let PropertyAttempt::Failed(diagnostic) = outcome else { + panic!("the deliberately different boxed answer should fail"); + }; + assert_eq!(diagnostic.code, deed_mir::codes::ASSERTION_FAILED); + } + + #[test] + fn a_corrupt_compiled_list_length_is_not_allocated_by_the_runner() { + let memory = i64::MAX.to_le_bytes(); + assert!( + decode_property_word( + &deed_mir::Program::new(), + &deed_mir::Ty::List(Box::new(deed_mir::Ty::Int)), + 0, + &memory, + None, + &[(); 65], + ) + .is_none() + ); + } + + #[test] + fn a_list_that_exactly_fills_the_remaining_memory_is_read() { + let mut memory = Vec::new(); + memory.extend_from_slice(&1i64.to_le_bytes()); + memory.extend_from_slice(&7i64.to_le_bytes()); + assert_eq!( + decode_property_word( + &deed_mir::Program::new(), + &deed_mir::Ty::List(Box::new(deed_mir::Ty::Int)), + 0, + &memory, + None, + &[(); 65], + ), + Some(deed_interp::Value::List(std::rc::Rc::new(vec![ + deed_interp::Value::Int(7), + ]))) + ); + } } diff --git a/crates/deed-cli/tests/cli.rs b/crates/deed-cli/tests/cli.rs index 73a6b67..7b68d65 100644 --- a/crates/deed-cli/tests/cli.rs +++ b/crates/deed-cli/tests/cli.rs @@ -266,6 +266,64 @@ fn checking_the_worked_example_is_silent_and_succeeds() { #[test] fn compiled_tests_report_the_same_complete_count_as_the_interpreter() { + let scratch = Scratch::new("compiled-property-count"); + let file = scratch.write( + "properties.deed", + "module a\n\n\ + choice Tone {\n Soft,\n Hard,\n}\n\n\ + choice Mood {\n Calm,\n Loud { by: Int, tone: Tone },\n}\n\n\ + record Sample {\n numbers: List,\n moods: List,\n word: String,\n mood: Mood,\n yes: Bool,\n nothing: (),\n}\n\n\ + fn same(n: Int) -> Int\n\ + ensures ok => result == n,\n\ + { n }\n\n\ + fn maybe_one() -> Result\n\ + ensures ok => result == 1,\n\ + { ok(1) }\n\n\ + fn same_sample(sample: Sample) -> Sample\n\ + ensures ok => result == sample,\n\ + { sample }\n\n\ + fn same_result(value: Result) -> Result\n\ + ensures\n\ + ok => result == result,\n\ + err => result == result,\n\ + { value }\n\n\ + fn same_unit(value: ()) -> ()\n\ + ensures ok => result == result,\n\ + { value }\n\n\ + fn same_bool(value: Bool) -> Bool\n\ + ensures ok => result == value,\n\ + { value }\n", + ); + + let interpreted = run(&["test", file.to_str().unwrap()]); + let compiled = run(&["test", "--compiled", file.to_str().unwrap()]); + assert_eq!( + code(&interpreted), + 0, + "{}{}", + stdout(&interpreted), + stderr(&interpreted) + ); + assert_eq!( + code(&compiled), + 0, + "{}{}", + stdout(&compiled), + stderr(&compiled) + ); + assert_eq!(stdout(&compiled), stdout(&interpreted)); + assert!( + stdout(&compiled).contains("6 passed, 0 failed"), + "{}", + stdout(&compiled) + ); + assert!(stdout(&compiled).contains("property maybe_one (100 cases)")); + assert!(stdout(&compiled).contains("property same_result (100 cases)")); + assert!(stdout(&compiled).contains("property same_unit (100 cases)")); +} + +#[test] +fn compiled_written_tests_keep_their_own_count() { let output = run(&[ "test", "--compiled", @@ -277,6 +335,76 @@ fn compiled_tests_report_the_same_complete_count_as_the_interpreter() { "{}", stdout(&output) ); + assert!(!stdout(&output).contains("no tests found")); +} + +#[test] +fn compiled_assert_refuses_accepts_contract_failures_and_not_assertions() { + let scratch = Scratch::new("compiled-refuses-kind"); + let accepted = scratch.write( + "accepted.deed", + "module accepted\n\n\ + fn positive(n: Int) -> Int where n > 0, { n }\n\n\ + test \"a contract refusal counts\" {\n\ + assert refuses positive(0)\n\ + }\n", + ); + let rejected = scratch.write( + "rejected.deed", + "module rejected\n\n\ + fn asserts() -> Int {\n\ + assert false\n\ + 0\n\ + }\n\n\ + test \"an assertion is not a refusal\" {\n\ + assert refuses asserts()\n\ + }\n", + ); + + let accepted = run(&["test", "--compiled", accepted.to_str().unwrap()]); + assert_eq!( + code(&accepted), + 0, + "{}{}", + stdout(&accepted), + stderr(&accepted) + ); + assert!(stdout(&accepted).contains("1 passed, 0 failed")); + + let rejected = run(&["test", "--compiled", rejected.to_str().unwrap()]); + assert_eq!( + code(&rejected), + 1, + "{}{}", + stdout(&rejected), + stderr(&rejected) + ); + assert!( + stdout(&rejected).contains("assert refuses` probe trapped unexpectedly"), + "{}", + stdout(&rejected) + ); +} + +#[test] +fn compiled_tests_run_the_properties_contracts_generate() { + let scratch = Scratch::new("compiled-property"); + let file = scratch.write( + "property.deed", + "module a\n\n\ + fn wrong(n: Int) -> Int\n\ + ensures ok => result == n,\n\ + { n + 1 }\n\n\ + test \"the written block passes\" {\n assert true\n}\n", + ); + + let output = run(&["test", "--compiled", file.to_str().unwrap()]); + assert_eq!(code(&output), 1, "{}{}", stdout(&output), stderr(&output)); + assert!( + stdout(&output).contains("FAIL property wrong"), + "{}", + stdout(&output) + ); } #[test] diff --git a/crates/deed-codegen/src/lib.rs b/crates/deed-codegen/src/lib.rs index 86841c9..ac4d039 100644 --- a/crates/deed-codegen/src/lib.rs +++ b/crates/deed-codegen/src/lib.rs @@ -28,7 +28,8 @@ pub mod wasm; pub use compile::{Unsupported, compile, escaping_operations}; pub use grant::{Granted, Grants}; pub use run::{ - Host, HostCall, LinkError, Linked, Outcome, Trap, Value, call, call_measured, call_within, + Host, HostCall, LinkError, Linked, Outcome, Trap, Value, call, call_measured, call_prepared, + call_within, }; pub use validate::{Invalid, validate}; pub use wasm::Module; diff --git a/crates/deed-codegen/src/run.rs b/crates/deed-codegen/src/run.rs index dd22237..4927bfe 100644 --- a/crates/deed-codegen/src/run.rs +++ b/crates/deed-codegen/src/run.rs @@ -450,6 +450,20 @@ pub fn call(module: &Module, name: &str, args: &[Value]) -> Result Ok(call_measured(module, name, args)?.value) } +/// Calls an export after letting the caller build its arguments in the +/// module's own memory. +pub fn call_prepared( + module: &Module, + name: &str, + prepare: impl FnOnce(&mut HostCall<'_>) -> Result, Trap>, +) -> Result<(Option, Vec), Trap> { + let index = validated_export(module, name)?; + let mut memory = memory_of(module); + let args = prepare(&mut HostCall::new(&[], &mut memory))?; + let (outcome, memory) = run_export(module, index, &args, BUDGET, memory)?; + Ok((outcome.value, memory)) +} + /// Calls an exported function and reports what it allocated and ran. pub fn call_measured(module: &Module, name: &str, args: &[Value]) -> Result { call_within(module, name, args, BUDGET) @@ -468,21 +482,43 @@ pub fn call_within( args: &[Value], budget: u64, ) -> Result { + Ok(call_within_and_memory(module, name, args, budget)?.0) +} + +fn call_within_and_memory( + module: &Module, + name: &str, + args: &[Value], + budget: u64, +) -> Result<(Outcome, Vec), Trap> { + let index = validated_export(module, name)?; + run_export(module, index, args, budget, memory_of(module)) +} + +fn validated_export(module: &Module, name: &str) -> Result { if let Err(crate::validate::Invalid(reason)) = crate::validate::validate(module) { return Err(Trap::Invalid(reason)); } - let index = module + module .exports .iter() .find(|(exported, _)| exported == name) .map(|(_, index)| *index) - .ok_or_else(|| Trap::Unimplemented(format!("no export named {name}")))?; + .ok_or_else(|| Trap::Unimplemented(format!("no export named {name}"))) +} +fn run_export( + module: &Module, + index: u32, + args: &[Value], + budget: u64, + memory: Vec, +) -> Result<(Outcome, Vec), Trap> { let mut run = Run { module, fuel: budget, - memory: memory_of(module), + memory, host: None, }; let before = run.bump(); @@ -492,11 +528,14 @@ pub fn call_within( // in memory and memory is what this owns. Err(Trap::Unreachable) => Err(run.why(None).unwrap_or(Trap::Unreachable)), Err(other) => Err(other), - Ok(value) => Ok(Outcome { - value, - allocated: run.bump().saturating_sub(before), - steps: budget.saturating_sub(run.fuel), - }), + Ok(value) => { + let outcome = Outcome { + value, + allocated: run.bump().saturating_sub(before), + steps: budget.saturating_sub(run.fuel), + }; + Ok((outcome, run.memory)) + } } } diff --git a/crates/deed-interp/src/lib.rs b/crates/deed-interp/src/lib.rs index 79968d0..9065799 100644 --- a/crates/deed-interp/src/lib.rs +++ b/crates/deed-interp/src/lib.rs @@ -71,8 +71,8 @@ pub use interp::{ run_main_profiled_reaching, run_main_reaching, run_main_watched, run_tests, }; pub use property::{ - GeneratedInputs, PropertyConfig, PropertyOutcome, generate_inputs, is_testable, run_properties, - shrink_inputs, + GeneratedInputs, PropertyAttempt, PropertyConfig, PropertyInterpreter, PropertyOutcome, + generate_inputs, is_testable, run_properties, run_properties_with, shrink_inputs, }; pub use sandbox::Refused; pub use value::{Capability, Fields, Value, VariantValue}; diff --git a/crates/deed-interp/src/property.rs b/crates/deed-interp/src/property.rs index 1c467a1..47f3e76 100644 --- a/crates/deed-interp/src/property.rs +++ b/crates/deed-interp/src/property.rs @@ -61,6 +61,32 @@ impl PropertyOutcome { } } +/// What happened when one generated input was handed to a runtime. +pub enum PropertyAttempt { + Passed(Value), + /// The input violated a precondition, which makes the generator a bad + /// caller rather than the function a bad function. + Rejected, + Failed(Diagnostic), +} + +/// The reference implementation of contract checking for generated inputs. +pub struct PropertyInterpreter<'a> { + interp: Interp<'a>, +} + +impl<'a> PropertyInterpreter<'a> { + pub fn new(program: &Program<'a>, file: FileId) -> Self { + Self { + interp: Interp::make(program, file), + } + } + + pub fn attempt(&mut self, function: &'a FnDecl, args: &[Value]) -> PropertyAttempt { + attempt(&mut self.interp, function, args) + } +} + /// Inputs generated for one function. pub struct GeneratedInputs { /// Inputs that satisfied the function's preconditions. @@ -97,6 +123,30 @@ pub fn run_properties<'a>( resolutions: &'a Resolutions, config: PropertyConfig, ) -> Vec { + let mut interp = PropertyInterpreter::new(program, file); + run_properties_with( + program, + file, + module, + resolutions, + config, + |function, args| interp.attempt(function, args), + ) +} + +/// Runs generated properties, handing each input to the runtime supplied by +/// the caller. +pub fn run_properties_with<'a, F>( + program: &Program<'a>, + file: FileId, + module: &'a Module, + resolutions: &'a Resolutions, + config: PropertyConfig, + mut attempt: F, +) -> Vec +where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ module .items .iter() @@ -106,7 +156,17 @@ pub fn run_properties<'a>( } _ => None, }) - .map(|function| run_property(program, file, module, resolutions, function, config)) + .map(|function| { + run_property_with( + program, + file, + module, + resolutions, + function, + config, + &mut attempt, + ) + }) .collect() } @@ -136,8 +196,8 @@ pub fn generate_inputs<'a>( continue; }; match attempt(&mut interp, function, &args) { - Attempt::Passed => cases.push(args), - Attempt::Rejected | Attempt::Failed(_) => rejected += 1, + PropertyAttempt::Passed(_) => cases.push(args), + PropertyAttempt::Rejected | PropertyAttempt::Failed(_) => rejected += 1, } } @@ -193,19 +253,24 @@ where args } -fn run_property<'a>( +fn run_property_with<'a, F>( program: &Program<'a>, file: FileId, module: &'a Module, resolutions: &'a Resolutions, function: &'a FnDecl, config: PropertyConfig, -) -> PropertyOutcome { + attempt: &mut F, +) -> PropertyOutcome +where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ let types = TypeIndex::new(module, resolutions); let mut rng = Rng::new(config.seed); - let mut interp = Interp::make(program, file); + let mut generator = Interp::make(program, file); let mut cases = 0usize; + let mut attempts = 0usize; let mut rejected = 0usize; // A generator that keeps producing inputs the preconditions throw away is // not testing anything, so give up rather than run forever. @@ -216,17 +281,17 @@ fn run_property<'a>( break; } - let Some(args) = generate_arguments(&types, &mut rng, function, &mut interp) else { - rejected += 1; + let Some(args) = generate_arguments(&types, &mut rng, function, &mut generator) else { continue; }; - - match attempt(&mut interp, function, &args) { - Attempt::Passed => cases += 1, - Attempt::Rejected => rejected += 1, - Attempt::Failed(diagnostic) => { - let simpler = Simpler::of(&types, &mut interp); - let (args, diagnostic) = shrink(&mut interp, function, args, diagnostic, &simpler); + attempts += 1; + + match attempt(function, &args) { + PropertyAttempt::Passed(_) => cases += 1, + PropertyAttempt::Rejected => rejected = attempts - cases, + PropertyAttempt::Failed(diagnostic) => { + let simpler = Simpler::of(&types, &mut generator); + let (args, diagnostic) = shrink(function, args, diagnostic, &simpler, attempt); return PropertyOutcome { function: function.sig.name.name.clone(), span: function.sig.name.span, @@ -270,15 +335,7 @@ fn run_property<'a>( } } -enum Attempt { - Passed, - /// The input violated a precondition, which makes the generator a bad - /// caller rather than the function a bad function. - Rejected, - Failed(Diagnostic), -} - -fn attempt<'a>(interp: &mut Interp<'a>, function: &'a FnDecl, args: &[Value]) -> Attempt { +fn attempt<'a>(interp: &mut Interp<'a>, function: &'a FnDecl, args: &[Value]) -> PropertyAttempt { let call_args = args .iter() .cloned() @@ -286,9 +343,11 @@ fn attempt<'a>(interp: &mut Interp<'a>, function: &'a FnDecl, args: &[Value]) -> .collect(); match interp.call_from_outside(function, call_args, function.sig.name.span) { - Ok(_) => Attempt::Passed, - Err(diagnostic) if diagnostic.code == codes::PRECONDITION_FAILED => Attempt::Rejected, - Err(diagnostic) => Attempt::Failed(*diagnostic), + Ok(value) => PropertyAttempt::Passed(value), + Err(diagnostic) if diagnostic.code == codes::PRECONDITION_FAILED => { + PropertyAttempt::Rejected + } + Err(diagnostic) => PropertyAttempt::Failed(*diagnostic), } } @@ -305,24 +364,27 @@ fn attempt<'a>(interp: &mut Interp<'a>, function: &'a FnDecl, args: &[Value]) -> /// counterexample built out of something nothing shrinks is a counterexample /// nobody reads, and it looks exactly like a small one that happens to be /// awkward. -fn shrink<'a>( - interp: &mut Interp<'a>, +fn shrink<'a, F>( function: &'a FnDecl, mut args: Vec, mut failure: Diagnostic, simpler: &Simpler, -) -> (Vec, Diagnostic) { + attempt: &mut F, +) -> (Vec, Diagnostic) +where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ let mut budget = 300usize; for index in 0..args.len() { if matches!(args[index], Value::Int(_)) { shrink_int( - interp, function, &mut args, index, &mut failure, &mut budget, + attempt, ); } } @@ -340,7 +402,7 @@ fn shrink<'a>( let mut attempt_args = args.clone(); attempt_args[index] = candidate; - if let Attempt::Failed(diagnostic) = attempt(interp, function, &attempt_args) { + if let PropertyAttempt::Failed(diagnostic) = attempt(function, &attempt_args) { args = attempt_args; failure = diagnostic; continue 'outer; @@ -353,14 +415,16 @@ fn shrink<'a>( (args, failure) } -fn shrink_int<'a>( - interp: &mut Interp<'a>, +fn shrink_int<'a, F>( function: &'a FnDecl, args: &mut [Value], index: usize, failure: &mut Diagnostic, budget: &mut usize, -) { + attempt: &mut F, +) where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ let Value::Int(start) = args[index] else { return; }; @@ -368,11 +432,17 @@ fn shrink_int<'a>( return; } - let sign = if start > 0 { 1i64 } else { -1 }; + let signed = |magnitude: i64| { + if start.is_negative() { + 0i64.saturating_sub(magnitude) + } else { + magnitude + } + }; let magnitude = start.checked_abs().unwrap_or(i64::MAX); // If zero fails too, there is no smaller counterexample to look for. - if let Some(diagnostic) = try_value(interp, function, args, index, 0, budget) { + if let Some(diagnostic) = try_value(function, args, index, 0, budget, attempt) { args[index] = Value::Int(0); *failure = diagnostic; return; @@ -381,9 +451,12 @@ fn shrink_int<'a>( let mut good = 0i64; let mut bad = magnitude; - while bad - good > 1 && *budget > 0 { + for _ in 0..*budget { + if bad - good == 1 { + break; + } let middle = good + (bad - good) / 2; - match try_value(interp, function, args, index, middle * sign, budget) { + match try_value(function, args, index, signed(middle), budget, attempt) { Some(diagnostic) => { bad = middle; *failure = diagnostic; @@ -392,36 +465,38 @@ fn shrink_int<'a>( } } - args[index] = Value::Int(bad * sign); - if let Some(diagnostic) = attempt_with(interp, function, args) { + args[index] = Value::Int(signed(bad)); + if let Some(diagnostic) = attempt_with(function, args, attempt) { *failure = diagnostic; } } /// Runs the function with one argument replaced, restoring it afterwards. -fn try_value<'a>( - interp: &mut Interp<'a>, +fn try_value<'a, F>( function: &'a FnDecl, args: &mut [Value], index: usize, candidate: i64, budget: &mut usize, -) -> Option { + attempt: &mut F, +) -> Option +where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ *budget = budget.saturating_sub(1); let original = args[index].clone(); args[index] = Value::Int(candidate); - let outcome = attempt_with(interp, function, args); + let outcome = attempt_with(function, args, attempt); args[index] = original; outcome } -fn attempt_with<'a>( - interp: &mut Interp<'a>, - function: &'a FnDecl, - args: &[Value], -) -> Option { - match attempt(interp, function, args) { - Attempt::Failed(diagnostic) => Some(diagnostic), +fn attempt_with<'a, F>(function: &'a FnDecl, args: &[Value], attempt: &mut F) -> Option +where + F: FnMut(&'a FnDecl, &[Value]) -> PropertyAttempt, +{ + match attempt(function, args) { + PropertyAttempt::Failed(diagnostic) => Some(diagnostic), _ => None, } } diff --git a/crates/deed-interp/tests/properties.rs b/crates/deed-interp/tests/properties.rs index b32592a..5246733 100644 --- a/crates/deed-interp/tests/properties.rs +++ b/crates/deed-interp/tests/properties.rs @@ -215,6 +215,24 @@ fn the_counterexample_is_shrunk_to_something_readable() { ); } +#[test] +fn a_negative_counterexample_is_shrunk_to_the_nearest_boundary() { + let (sources, outcome) = only( + "module a\n\n\ + fn not_too_low(n: Int) -> Int\n\ + \x20 where n < 0,\n\ + \x20 ensures ok => result > 0 - 50,\n\ + { n }\n", + ); + + let failure = outcome.failure.as_ref().expect("should have been caught"); + let text = render_human(&sources, failure); + assert!( + text.contains("generated input: n = -50"), + "expected the nearest failing input, got:\n{text}" + ); +} + /// The one counterexample from `src`, rendered. fn counterexample(src: &str) -> String { let (sources, outcome) = only(src); @@ -397,10 +415,7 @@ fn inputs_violating_a_precondition_are_discarded_not_reported() { ); assert!(outcome.passed(), "{:?}", outcome.failure.map(|f| f.message)); assert_eq!(outcome.cases, 100); - assert!( - outcome.rejected > 0, - "roughly half of the inputs should be discarded" - ); + assert_eq!(outcome.rejected, 135); } #[test]