From 7b1118dbee9226507747aa5cf244c3665bcc3ee7 Mon Sep 17 00:00:00 2001 From: Alec Thilenius Date: Tue, 2 May 2023 10:03:11 -0700 Subject: [PATCH] Add auto-downloading of protoc, update readme --- .vscode/settings.json | 3 + README.md | 55 ++++++++++++------ axum-connect-build/Cargo.toml | 3 +- axum-connect-build/src/lib.rs | 75 +++++++++++++++++++++++-- axum-connect-examples/build.rs | 6 +- axum-connect-examples/proto/hello.proto | 2 +- axum-connect-examples/src/main.rs | 3 +- 7 files changed, 121 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 60c622c..411cacf 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "cSpell.words": [ "bufbuild", "codegen", + "DTLS", "impls", "pbjson", "prost", @@ -9,7 +10,9 @@ "protobuf", "protoc", "protos", + "Roadmap", "serde", + "SRTP", "Thilenius", "typecheck" ] diff --git a/README.md b/README.md index c5683a4..0f9f8e4 100644 --- a/README.md +++ b/README.md @@ -34,24 +34,14 @@ _Prior knowledge with [Protobuf](https://github.com/protocolbuffers/protobuf) (both the IDL and it's use in RPC frameworks) and [Axum](https://github.com/tokio-rs/axum) are assumed._ -## Dependencies 🙄 +## Dependencies 👀 -Axum-connect uses [prost](https://github.com/tokio-rs/prost) for much of it's -protobuf manipulation. Prost sopped shipping `protoc` so you'll need to install -that manually, see the [grpc protoc install -instruction](https://grpc.io/docs/protoc-installation/). Alternatively there are -crates that ship pre-built protoc binaries. - -```sh -# On Debian systems -sudo apt install protobuf-compiler -``` - -You'll need 2 axum-connect crates, one for code-gen and one for runtime use. +You'll need 2 `axum-connect` crates, one for code-gen and one for runtime use. Because of how prost works, you'll also need to add it to your own project. You'll obviously also need `axum` and `tokio`. ```sh +# Note: axum-connect-build will fetch `protoc` for you. cargo add --build axum-connect-build cargo add axum-connect prost axum cargo add tokio --features full @@ -88,10 +78,15 @@ Use the `axum_connect_codegen` crate to generate Rust code from the proto IDL. `build.rs` ```rust -use axum_connect_build::axum_connect_codegen; +use axum_connect_build::{axum_connect_codegen, AxumConnectGenSettings}; fn main() { - axum_connect_codegen(&["proto"], &["proto/hello.proto"]).unwrap(); + // This helper will use `proto` as the import path, and globs all .proto + // files in the `proto` directory. You can build an AxumConnectGenSettings + // manually too, if you wish. + let settings = AxumConnectGenSettings::from_directory_recursive("proto") + .expect("failed to glob proto files"); + axum_connect_codegen(settings).unwrap(); } ``` @@ -170,7 +165,7 @@ API with end-to-end typed RPCs. - Possibly maybe-someday support BiDi streaming over WebRTC - This would require `connect-web` picking up support for the same - WebRTC streams because they are DTLS/SRTP and are resilient -- Replace Prost +- Replace Prost (with something custom and simpler) ## Non-goals @@ -184,6 +179,34 @@ API with end-to-end typed RPCs. - This is idiomatic Rust. Do one thing well, and leave the rest to other crates. +# Prost and Protobuf + +## Protoc Version + +The installed version of `protoc` can be configured in the +`AxumConnectGenSettings` if you need/wish to do so. Setting the value to `None` +will disable the download entirely. + +## Reasoning + +Prost stopped shipping `protoc` binaries (a decision I disagree with) so +`axum-connect-build` internally uses +[protoc-fetcher](https://crates.io/crates/protoc-fetcher) download and resolve a +copy of `protoc`. This is far more turnkey than forcing every build environment +(often Heroku and/or Docker) to have a recent `protoc` binary pre-installed. + +Prost removed it in the name of security, but I fail to see how executing a +hash-checked crate as part of your build is any less dangerous than executing a +hash-checked binary as part of your build. Both get to run binary code on your +build machine. This behavior can be disabled of you disagree, if you need to +comply with corporate policy, or your build environment is offline. + +I would someday like to replace all of it with a new 'lean and +mean' protoc library for the Rust community. One with a built-in parser, that +supports only the latest proto3 syntax as well as the canonical JSON +serialization format and explicitly doesn't support many of the rarely used +features. But that day is not today. + # Versioning `axum-connect` and `axum-connect-build` versions are currently **not** kept in diff --git a/axum-connect-build/Cargo.toml b/axum-connect-build/Cargo.toml index 9a4def0..a43f920 100644 --- a/axum-connect-build/Cargo.toml +++ b/axum-connect-build/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "axum-connect-build" -version = "0.1.2" +version = "0.1.3" authors = ["Alec Thilenius "] edition = "2021" categories = [ @@ -22,5 +22,6 @@ proc-macro2 = "1.0.56" prost = "0.11.9" prost-build = "0.11.9" prost-reflect = "0.11.4" +protoc-fetcher = "0.1.0" quote = "1.0.26" syn = "2.0.15" diff --git a/axum-connect-build/src/lib.rs b/axum-connect-build/src/lib.rs index 3212f23..64eb813 100644 --- a/axum-connect-build/src/lib.rs +++ b/axum-connect-build/src/lib.rs @@ -8,18 +8,83 @@ use gen::AxumConnectServiceGenerator; mod gen; -pub fn axum_connect_codegen( - include: &[impl AsRef], - inputs: &[impl AsRef], -) -> anyhow::Result<()> { +#[derive(Clone, Debug)] +pub struct AxumConnectGenSettings { + pub includes: Vec, + pub inputs: Vec, + pub protoc_args: Vec, + pub protoc_version: Option, +} + +impl Default for AxumConnectGenSettings { + fn default() -> Self { + Self { + includes: Default::default(), + inputs: Default::default(), + protoc_args: Default::default(), + protoc_version: Some("22.3".to_string()), + } + } +} + +impl AxumConnectGenSettings { + pub fn from_directory_recursive

(path: P) -> anyhow::Result + where + P: Into, + { + let path = path.into(); + let mut settings = Self::default(); + settings.includes.push(path.clone()); + + // Recursively add all files that end in ".proto" to the inputs. + let mut dirs = vec![path]; + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + dirs.push(path.clone()); + } else if path.extension().map(|ext| ext == "proto").unwrap_or(false) { + settings.inputs.push(path); + } + } + } + + Ok(settings) + } +} + +pub fn axum_connect_codegen(settings: AxumConnectGenSettings) -> anyhow::Result<()> { + // Fetch protoc + if let Some(version) = &settings.protoc_version { + let out_dir = env::var("OUT_DIR").unwrap(); + let protoc_path = protoc_fetcher::protoc(version, Path::new(&out_dir))?; + env::set_var("PROTOC", protoc_path); + } + + // Instruct cargo to re-run if any of the proto files change + for input in &settings.inputs { + println!("cargo:rerun-if-changed={}", input.display()); + } + let descriptor_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("proto_descriptor.bin"); let mut conf = prost_build::Config::new(); + + // Standard prost configuration conf.compile_well_known_types(); conf.file_descriptor_set_path(&descriptor_path); conf.extern_path(".google.protobuf", "::pbjson_types"); conf.service_generator(Box::new(AxumConnectServiceGenerator::new())); - conf.compile_protos(inputs, include).unwrap(); + + // Arg configuration + for arg in settings.protoc_args { + conf.protoc_arg(arg); + } + + // File configuration + conf.compile_protos(&settings.inputs, &settings.includes) + .unwrap(); // Use pbjson to generate the Serde impls, and inline them with the Prost files. let descriptor_set = std::fs::read(descriptor_path)?; diff --git a/axum-connect-examples/build.rs b/axum-connect-examples/build.rs index 71002bb..fe9bf5f 100644 --- a/axum-connect-examples/build.rs +++ b/axum-connect-examples/build.rs @@ -1,5 +1,7 @@ -use axum_connect_build::axum_connect_codegen; +use axum_connect_build::{axum_connect_codegen, AxumConnectGenSettings}; fn main() { - axum_connect_codegen(&["proto"], &["proto/hello.proto"]).unwrap(); + let settings = AxumConnectGenSettings::from_directory_recursive("proto") + .expect("failed to glob proto files"); + axum_connect_codegen(settings).unwrap(); } diff --git a/axum-connect-examples/proto/hello.proto b/axum-connect-examples/proto/hello.proto index 7004699..826d4cd 100644 --- a/axum-connect-examples/proto/hello.proto +++ b/axum-connect-examples/proto/hello.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package hello; -message HelloRequest { string name = 1; } +message HelloRequest { optional string name = 1; } message HelloResponse { string message = 1; } diff --git a/axum-connect-examples/src/main.rs b/axum-connect-examples/src/main.rs index 630b245..253113d 100644 --- a/axum-connect-examples/src/main.rs +++ b/axum-connect-examples/src/main.rs @@ -30,7 +30,8 @@ async fn say_hello_success(Host(host): Host, request: HelloRequest) -> HelloResp HelloResponse { message: format!( "Hello {}! You're addressing the hostname: {}.", - request.name, host + request.name.unwrap_or_else(|| "unnamed".to_string()), + host ), } }