diff --git a/Cargo.lock b/Cargo.lock index 402c1541f..96a8dcd9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -982,6 +982,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codegen" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "573800db6c3319bc125ddbf9b9cb001ad1602957f53642ba8d09ff3ddd4da7f1" +dependencies = [ + "indexmap", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -3029,6 +3038,7 @@ name = "miden-node-proto-build" version = "0.14.0" dependencies = [ "build-rs", + "codegen", "fs-err", "miette", "protox", diff --git a/Cargo.toml b/Cargo.toml index 3e41e9c1d..8f441f239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ assert_matches = { version = "1.5" } async-trait = { version = "0.1" } build-rs = { version = "0.3" } clap = { features = ["derive"], version = "4.5" } +codegen = { version = "0.3" } deadpool = { default-features = false, version = "0.12" } deadpool-diesel = { version = "0.6" } deadpool-sync = { default-features = false, version = "0.1" } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index 07117f466..3e6e037b3 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -5,9 +5,7 @@ use miden_node_proto_build::{ block_producer_api_descriptor, remote_prover_api_descriptor, rpc_api_descriptor, - store_block_producer_api_descriptor, - store_ntx_builder_api_descriptor, - store_rpc_api_descriptor, + store_api_descriptor, validator_api_descriptor, }; use miette::{Context, IntoDiagnostic}; @@ -26,9 +24,7 @@ fn main() -> miette::Result<()> { .wrap_err("creating destination folder")?; generate_bindings(rpc_api_descriptor(), &dst_dir)?; - generate_bindings(store_rpc_api_descriptor(), &dst_dir)?; - generate_bindings(store_ntx_builder_api_descriptor(), &dst_dir)?; - generate_bindings(store_block_producer_api_descriptor(), &dst_dir)?; + generate_bindings(store_api_descriptor(), &dst_dir)?; generate_bindings(block_producer_api_descriptor(), &dst_dir)?; generate_bindings(remote_prover_api_descriptor(), &dst_dir)?; generate_bindings(validator_api_descriptor(), &dst_dir)?; diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 8c828f116..e355aae7a 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -5,11 +5,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::generated::store; -use miden_node_proto_build::{ - store_block_producer_api_descriptor, - store_ntx_builder_api_descriptor, - store_rpc_api_descriptor, -}; +use miden_node_proto_build::store_api_descriptor; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::signer::BlockSigner; use miden_node_utils::tracing::grpc::grpc_trace_fn; @@ -126,9 +122,7 @@ impl Store { block_prover: Arc::clone(&block_prover), }); let reflection_service = tonic_reflection::server::Builder::configure() - .register_file_descriptor_set(store_rpc_api_descriptor()) - .register_file_descriptor_set(store_ntx_builder_api_descriptor()) - .register_file_descriptor_set(store_block_producer_api_descriptor()) + .register_file_descriptor_set(store_api_descriptor()) .build_v1() .context("failed to build reflection service")?; @@ -137,9 +131,7 @@ impl Store { // // See: . let reflection_service_alpha = tonic_reflection::server::Builder::configure() - .register_file_descriptor_set(store_rpc_api_descriptor()) - .register_file_descriptor_set(store_ntx_builder_api_descriptor()) - .register_file_descriptor_set(store_block_producer_api_descriptor()) + .register_file_descriptor_set(store_api_descriptor()) .build_v1alpha() .context("failed to build reflection service")?; diff --git a/proto/Cargo.toml b/proto/Cargo.toml index ee79d7adc..95ddd8ef8 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -25,6 +25,11 @@ tonic-prost-build = { workspace = true } [build-dependencies] build-rs = { workspace = true } +codegen = { workspace = true } fs-err = { workspace = true } miette = { version = "7.6" } protox = { workspace = true } + +[package.metadata.cargo-machete] +# Machete misses these because they're required in files generated by build.rs. +ignored = ["protox", "tonic-prost-build"] diff --git a/proto/build.rs b/proto/build.rs index 7246ab495..cb5c038e3 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -1,79 +1,105 @@ +use std::ffi::OsStr; +use std::path::PathBuf; + use fs_err as fs; -use miette::{Context, IntoDiagnostic}; +use miette::{IntoDiagnostic, miette}; use protox::prost::Message; -const RPC_PROTO: &str = "rpc.proto"; -// Unified internal store API (store.Rpc, store.BlockProducer, store.NtxBuilder). -// We compile the same file three times to preserve existing descriptor names. -const STORE_RPC_PROTO: &str = "internal/store.proto"; -const STORE_NTX_BUILDER_PROTO: &str = "internal/store.proto"; -const STORE_BLOCK_PRODUCER_PROTO: &str = "internal/store.proto"; -const BLOCK_PRODUCER_PROTO: &str = "internal/block_producer.proto"; -const REMOTE_PROVER_PROTO: &str = "remote_prover.proto"; -const VALIDATOR_PROTO: &str = "internal/validator.proto"; - -const RPC_DESCRIPTOR: &str = "rpc_file_descriptor.bin"; -const STORE_RPC_DESCRIPTOR: &str = "store_rpc_file_descriptor.bin"; -const STORE_NTX_BUILDER_DESCRIPTOR: &str = "store_ntx_builder_file_descriptor.bin"; -const STORE_BLOCK_PRODUCER_DESCRIPTOR: &str = "store_block_producer_file_descriptor.bin"; -const BLOCK_PRODUCER_DESCRIPTOR: &str = "block_producer_file_descriptor.bin"; -const REMOTE_PROVER_DESCRIPTOR: &str = "remote_prover_file_descriptor.bin"; -const VALIDATOR_DESCRIPTOR: &str = "validator_file_descriptor.bin"; - -/// Generates Rust protobuf bindings from .proto files. +/// Compiles each gRPC service definitions into a +/// [`FileDescriptorSet`](tonic_prost_build::FileDescriptorSet) and exposes it as a function: /// -/// This is done only if `BUILD_PROTO` environment variable is set to `1` to avoid running the -/// script on crates.io where repo-level .proto files are not available. +/// ```rust +/// fn _api_descriptor() -> FileDescriptorSet; +/// ``` fn main() -> miette::Result<()> { build_rs::output::rerun_if_changed("./proto"); + build_rs::output::rerun_if_changed("Cargo.toml"); let out_dir = build_rs::input::out_dir(); - let crate_root = build_rs::input::cargo_manifest_dir(); - let proto_src_dir = crate_root.join("proto"); - let includes = &[proto_src_dir]; - - let rpc_file_descriptor = protox::compile([RPC_PROTO], includes)?; - let rpc_path = out_dir.join(RPC_DESCRIPTOR); - fs::write(&rpc_path, rpc_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing rpc file descriptor")?; - - let remote_prover_file_descriptor = protox::compile([REMOTE_PROVER_PROTO], includes)?; - let remote_prover_path = out_dir.join(REMOTE_PROVER_DESCRIPTOR); - fs::write(&remote_prover_path, remote_prover_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing remote prover file descriptor")?; - - let store_rpc_file_descriptor = protox::compile([STORE_RPC_PROTO], includes)?; - let store_rpc_path = out_dir.join(STORE_RPC_DESCRIPTOR); - fs::write(&store_rpc_path, store_rpc_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing store rpc file descriptor")?; - - let store_ntx_builder_file_descriptor = protox::compile([STORE_NTX_BUILDER_PROTO], includes)?; - let store_ntx_builder_path = out_dir.join(STORE_NTX_BUILDER_DESCRIPTOR); - fs::write(&store_ntx_builder_path, store_ntx_builder_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing store ntx builder file descriptor")?; - - let store_block_producer_file_descriptor = - protox::compile([STORE_BLOCK_PRODUCER_PROTO], includes)?; - let store_block_producer_path = out_dir.join(STORE_BLOCK_PRODUCER_DESCRIPTOR); - fs::write(&store_block_producer_path, store_block_producer_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing store block producer file descriptor")?; - - let block_producer_file_descriptor = protox::compile([BLOCK_PRODUCER_PROTO], includes)?; - let block_producer_path = out_dir.join(BLOCK_PRODUCER_DESCRIPTOR); - fs::write(&block_producer_path, block_producer_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing block producer file descriptor")?; - - let validator_file_descriptor = protox::compile([VALIDATOR_PROTO], includes)?; - let validator_path = out_dir.join(VALIDATOR_DESCRIPTOR); - fs::write(&validator_path, validator_file_descriptor.encode_to_vec()) - .into_diagnostic() - .wrap_err("writing validator file descriptor")?; + let schema_dir = build_rs::input::cargo_manifest_dir().join("proto"); + + // Codegen which will hold the file descriptor functions. + // + // `protox::prost::Message` is a trait which brings into scope the encoding and decoding of file + // descriptors. This is required so because we serialize the descriptors in code as a `Vec` + // and then decode it again inline. + let mut code = codegen::Scope::new(); + code.import("tonic_prost_build", "FileDescriptorSet"); + code.import("protox::prost", "Message"); + + // We split our gRPC services into public and internal. + // + // This is easy to do since public services are listed in the root of the schema folder, + // and internal services are nested in the `internal` folder. + for public_api in proto_files_in_directory(&schema_dir)? { + let file_descriptor_fn = generate_file_descriptor(&public_api, &schema_dir)?; + code.push_fn(file_descriptor_fn); + } + + // Internal gRPC services need an additional feature gate `#[cfg(feature = "internal")]`. + for internal_api in proto_files_in_directory(&schema_dir.join("internal"))? { + let mut file_descriptor_fn = generate_file_descriptor(&internal_api, &schema_dir)?; + file_descriptor_fn.attr("cfg(feature = \"internal\")"); + code.push_fn(file_descriptor_fn); + } + + fs::write(out_dir.join("file_descriptors.rs"), code.to_string()).into_diagnostic()?; Ok(()) } + +/// The list of `*.proto` files in the given directory. +/// +/// Does _not_ recurse into folders; only top level files are returned. +fn proto_files_in_directory(directory: &PathBuf) -> Result, miette::Error> { + let mut proto_files = Vec::new(); + for entry in fs::read_dir(directory).into_diagnostic()? { + let entry = entry.into_diagnostic()?; + + // Skip non-files + if !entry.file_type().into_diagnostic()?.is_file() { + continue; + } + + // Skip non-protobuf files + if PathBuf::from(entry.file_name()).extension().is_none_or(|ext| ext != "proto") { + continue; + } + + proto_files.push(entry.path()); + } + Ok(proto_files) +} + +/// Creates a function which emits the file descriptor of the given gRPC service file. +/// +/// The function looks as follows: +/// +/// ```rust +/// fn _api_descriptor() -> FileDescriptorSet { +/// FileDescriptorSet::decode(vec![].as_slice()) +/// .expect("encoded file descriptor should decode") +/// } +/// ``` +/// +/// where `` is bytes of the compiled gRPC service. +fn generate_file_descriptor( + grpc_service: &PathBuf, + includes: &PathBuf, +) -> Result { + let file_name = grpc_service + .file_stem() + .and_then(OsStr::to_str) + .ok_or_else(|| miette!("invalid file name for {grpc_service:?}"))?; + + let file_descriptor = protox::compile([grpc_service], includes)?; + let file_descriptor = file_descriptor.encode_to_vec(); + + let mut f = codegen::Function::new(format!("{file_name}_api_descriptor")); + f.vis("pub") + .ret("FileDescriptorSet") + .line(format!("FileDescriptorSet::decode(vec!{file_descriptor:?}.as_slice())")) + .line(".expect(\"we just encoded this so it should decode\")"); + + Ok(f) +} diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 8e8440d19..f38cc3ad5 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,57 +1 @@ -use protox::prost::Message; -use tonic_prost_build::FileDescriptorSet; - -/// Returns the Protobuf file descriptor for the RPC API. -pub fn rpc_api_descriptor() -> FileDescriptorSet { - let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/", "rpc_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the remote prover API. -pub fn remote_prover_api_descriptor() -> FileDescriptorSet { - let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/", "remote_prover_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the store RPC API. -#[cfg(feature = "internal")] -pub fn store_rpc_api_descriptor() -> FileDescriptorSet { - let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/", "store_rpc_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the store NTX builder API. -#[cfg(feature = "internal")] -pub fn store_ntx_builder_api_descriptor() -> FileDescriptorSet { - let bytes = - include_bytes!(concat!(env!("OUT_DIR"), "/", "store_ntx_builder_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the store block producer API. -#[cfg(feature = "internal")] -pub fn store_block_producer_api_descriptor() -> FileDescriptorSet { - let bytes = - include_bytes!(concat!(env!("OUT_DIR"), "/", "store_block_producer_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the block-producer API. -#[cfg(feature = "internal")] -pub fn block_producer_api_descriptor() -> FileDescriptorSet { - let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/", "block_producer_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} - -/// Returns the Protobuf file descriptor for the validator API. -pub fn validator_api_descriptor() -> FileDescriptorSet { - let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/", "validator_file_descriptor.bin")); - FileDescriptorSet::decode(&bytes[..]) - .expect("bytes should be a valid file descriptor created by build.rs") -} +include!(concat!(env!("OUT_DIR"), "/file_descriptors.rs"));