Support for Unknown Fields (Rebase of #574)

README.md

@@ -287,6 +287,26 @@ pub mod foo {
 
 [^3]: Annotations have been elided for clarity. See below for a full example.
 
+#### Unknown Fields
+
+Protobuf's unknown fields are supported for round-trip consistency of messages. 
+See the [`prost-build` documentation][prost-build] for more details on generation.
+
+***IMPORTANT***: Note that unknown fields can potentially contain a Bytes object of undefined size. This violates compile-time size constraints, and thus means that any and all objects generated with unknown fields support enabled cannot implement the `Copy` trait.
+
+When using `prost`-generated objects that include unknown fields, all unknown fields are 
+stored in a field within the struct, generally named `unknown_fields`, although it can be
+modified in the `prost-build` build-script. If you are generating a new object of a type 
+that has been marked as having unknown fields, note that you will need to either import an existing
+`unknown_fields` object, or use a default constructor to your implementation, like so:
+```rust,ignore
+let val = MessageWithUnknownFields{
+  field_one: 1u32,
+  ..Default::default()
+};
+```
+This will initialise the `unknown_fields` as empty, and allow the object to be serialized correctly.
+
 ### Services
 
 `prost-build` allows a custom code-generator to be used for processing `service`

benchmarks/benches/dataset.rs

@@ -37,7 +37,7 @@ fn benchmark_dataset<M>(criterion: &mut Criterion, name: &str, dataset: &'static
 where
     M: prost::Message + Default + 'static,
 {
-    let mut group = criterion.benchmark_group(&format!("dataset/{}", name));
+    let mut group = criterion.benchmark_group(format!("dataset/{}", name));
 
     group.bench_function("merge", move |b| {
         let dataset = load_dataset(dataset).unwrap();

conformance/tests/conformance.rs

@@ -23,8 +23,6 @@ fn test_conformance() {
 
     let status = Command::new(conformance::test_runner())
         .arg("--enforce_recommended")
-        .arg("--failure_list")
-        .arg("failing_tests.txt")
         .arg(proto_conformance)
         .status()
         .expect("failed to execute conformance-test-runner");

prost-build/src/code_generator.rs

@@ -279,6 +279,14 @@ impl<'b> CodeGenerator<'_, 'b> {
             }
             self.path.pop();
         }
+        if let Some(field_name) = &self
+            .config()
+            .include_unknown_fields
+            .get_first(&fq_message_name)
+            .cloned()
+        {
+            self.append_unknown_field_set(&fq_message_name, field_name);
+        }
         self.path.pop();
 
         self.path.push(8);
@@ -581,6 +589,14 @@ impl<'b> CodeGenerator<'_, 'b> {
         ));
     }
 
+    fn append_unknown_field_set(&mut self, fq_message_name: &str, field_name: &str) {
+        self.buf.push_str("#[prost(unknown_fields)]\n");
+        self.append_field_attributes(fq_message_name, field_name);
+        self.push_indent();
+        self.buf
+            .push_str(&format!("pub {}: ::prost::UnknownFieldList,\n", field_name,));
+    }
+
     fn append_oneof_field(
         &mut self,
         message_name: &str,

prost-build/src/config.rs

@@ -36,6 +36,7 @@ pub struct Config {
     pub(crate) message_attributes: PathMap<String>,
     pub(crate) enum_attributes: PathMap<String>,
     pub(crate) field_attributes: PathMap<String>,
+    pub(crate) include_unknown_fields: PathMap<String>,
     pub(crate) boxed: PathMap<()>,
     pub(crate) prost_types: bool,
     pub(crate) strip_enum_prefix: bool,
@@ -266,6 +267,38 @@ impl Config {
         self
     }
 
+    /// Preserve unknown fields for the message type.
+    ///
+    /// # Arguments
+    ///
+    /// **`paths`** - paths to specific messages, or packages which should preserve unknown
+    /// fields during deserialization.
+    ///
+    /// **`override_field_name`** - the name of the field to place unknown fields in. A field 
+    /// with this name and type `prost::UnknownFieldList` will be added to the generated struct.
+    /// The default field used is named `unknown_fields`.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// # let mut config = prost_build::Config::new();
+    /// config.include_unknown_fields(".my_messages.MyMessageType", None::<String>);
+    /// config.include_unknown_fields(".my_messages.MyMessageType", Some("unique_field_name"));
+    /// ```
+    pub fn include_unknown_fields<P, A>(&mut self, path: P, override_field_name: Option<A>) -> &mut Self
+    where
+        P: AsRef<str>,
+        A: AsRef<str>,
+    {
+        let field_name:String = match override_field_name {
+            Some(x) => x.as_ref().to_string(),
+            None => "unknown_fields".to_string()
+        };
+        self.include_unknown_fields
+            .insert(path.as_ref().to_string(), field_name);
+        self
+    }
+
     /// Add additional attribute to matched messages.
     ///
     /// # Arguments
@@ -1202,6 +1235,7 @@ impl default::Default for Config {
             message_attributes: PathMap::default(),
             enum_attributes: PathMap::default(),
             field_attributes: PathMap::default(),
+            include_unknown_fields: PathMap::default(),
             boxed: PathMap::default(),
             prost_types: true,
             strip_enum_prefix: true,
@@ -1234,6 +1268,7 @@ impl fmt::Debug for Config {
             .field("bytes_type", &self.bytes_type)
             .field("type_attributes", &self.type_attributes)
             .field("field_attributes", &self.field_attributes)
+            .field("include_unknown_fields", &self.include_unknown_fields)
             .field("prost_types", &self.prost_types)
             .field("strip_enum_prefix", &self.strip_enum_prefix)
             .field("out_dir", &self.out_dir)

prost-build/src/context.rs

@@ -182,6 +182,16 @@ impl<'a> Context<'a> {
     /// Returns `true` if this message can automatically derive Copy trait.
     pub fn can_message_derive_copy(&self, fq_message_name: &str) -> bool {
         assert_eq!(".", &fq_message_name[..1]);
+        // Unknown fields can potentially include an unbounded Bytes object, which
+        // cannot implement Copy
+        if self
+            .config
+            .include_unknown_fields
+            .get_first(fq_message_name)
+            .is_some()
+        {
+            return false;
+        };
         self.message_graph
             .get_message(fq_message_name)
             .unwrap()

prost-build/src/lib.rs

@@ -80,6 +80,40 @@
 //! That's it! Run `cargo doc` to see documentation for the generated code. The full
 //! example project can be found on [GitHub](https://github.com/danburkert/snazzy).
 //!
+//! ## Unknown Fields
+//!
+//! `prost-build` supports unknown fields, however they need to be manually enabled in your
+//! `build.rs` build-script. This can be done by modifying the previously script like so:
+//! ```rust,no_run
+//! use std::io::Result;
+//! fn main() -> Result<()> {
+//!     //prost_build::compile_protos(&["src/items.proto"], &["src/"])?;
+//!     let mut config = prost_build::Config::new();
+//!     config.btree_map(["."]);
+//!     // To enable unknown fields for a single message:
+//!     config.include_unknown_fields(".snazzy.items.shirt", None::<String>);
+//!     // To enable unknown fields for a whole package:
+//!     config.include_unknown_fields(".snazzy.items", None::<String>);
+//!     config.compile_protos(&["src/items.proto"], &["src/"])?;
+//!     Ok(())
+//! }
+//! ```
+//!
+//! By default, this stores all unknown fields in a unique internal field called `unknown_fields`.
+//! If you already have a field in your message with this name, you can modify the above call like
+//! so:
+//! ```rust,no_run
+//! use std::io::Result;
+//! fn main() -> Result<()> {
+//!     //prost_build::compile_protos(&["src/items.proto"], &["src/"])?;
+//!     let mut config = prost_build::Config::new();
+//!     config.btree_map(["."]);
+//!     config.include_unknown_fields(".snazzy.items.shirt", Some("unused_unique_field_name"));
+//!     config.compile_protos(&["src/items.proto"], &["src/"])?;
+//!     Ok(())
+//! }
+//! ```
+//!
 //! ## Feature Flags
 //! - `format`: Format the generated output. This feature is enabled by default.
 //! - `cleanup-markdown`: Clean up Markdown in protobuf docs. Enable this to clean up protobuf files from third parties.

prost-derive/src/field/mod.rs

@@ -3,6 +3,7 @@ mod map;
 mod message;
 mod oneof;
 mod scalar;
+mod unknown;
 
 use std::fmt;
 use std::slice;
@@ -26,6 +27,8 @@ pub enum Field {
     Oneof(oneof::Field),
     /// A group field.
     Group(group::Field),
+    /// A set of unknown message fields.
+    Unknown(unknown::Field),
 }
 
 impl Field {
@@ -48,6 +51,8 @@ impl Field {
             Field::Oneof(field)
         } else if let Some(field) = group::Field::new(&attrs, inferred_tag)? {
             Field::Group(field)
+        } else if let Some(field) = unknown::Field::new(&attrs)? {
+            Field::Unknown(field)
         } else {
             bail!("no type attribute");
         };
@@ -86,6 +91,7 @@ impl Field {
             Field::Map(ref map) => vec![map.tag],
             Field::Oneof(ref oneof) => oneof.tags.clone(),
             Field::Group(ref group) => vec![group.tag],
+            Field::Unknown(_) => vec![],
         }
     }
 
@@ -97,6 +103,7 @@ impl Field {
             Field::Map(ref map) => map.encode(prost_path, ident),
             Field::Oneof(ref oneof) => oneof.encode(ident),
             Field::Group(ref group) => group.encode(prost_path, ident),
+            Field::Unknown(ref unknown) => unknown.encode(ident),
         }
     }
 
@@ -109,6 +116,7 @@ impl Field {
             Field::Map(ref map) => map.merge(prost_path, ident),
             Field::Oneof(ref oneof) => oneof.merge(ident),
             Field::Group(ref group) => group.merge(prost_path, ident),
+            Field::Unknown(ref unknown) => unknown.merge(ident),
         }
     }
 
@@ -120,6 +128,7 @@ impl Field {
             Field::Message(ref msg) => msg.encoded_len(prost_path, ident),
             Field::Oneof(ref oneof) => oneof.encoded_len(ident),
             Field::Group(ref group) => group.encoded_len(prost_path, ident),
+            Field::Unknown(ref unknown) => unknown.encoded_len(ident),
         }
     }
 
@@ -131,6 +140,7 @@ impl Field {
             Field::Map(ref map) => map.clear(ident),
             Field::Oneof(ref oneof) => oneof.clear(ident),
             Field::Group(ref group) => group.clear(ident),
+            Field::Unknown(ref unknown) => unknown.clear(ident),
         }
     }
 
@@ -173,6 +183,10 @@ impl Field {
             _ => None,
         }
     }
+
+    pub fn is_unknown(&self) -> bool {
+        matches!(self, Field::Unknown(_))
+    }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq)]

prost-derive/src/field/unknown.rs

@@ -0,0 +1,66 @@
+use anyhow::{bail, Error};
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::Meta;
+
+use crate::field::{set_bool, word_attr};
+
+#[derive(Clone)]
+pub struct Field {}
+
+impl Field {
+    pub fn new(attrs: &[Meta]) -> Result<Option<Field>, Error> {
+        let mut unknown = false;
+        let mut unknown_attrs = Vec::new();
+
+        for attr in attrs {
+            if word_attr("unknown_fields", attr) {
+                set_bool(&mut unknown, "duplicate message attribute")?;
+            } else {
+                unknown_attrs.push(attr);
+            }
+        }
+
+        if !unknown {
+            return Ok(None);
+        }
+
+        match unknown_attrs.len() {
+            0 => (),
+            1 => bail!(
+                "unknown attribute for unknown field set: {:?}",
+                unknown_attrs[0]
+            ),
+            _ => bail!(
+                "unknown attributes for unknown field set: {:?}",
+                unknown_attrs
+            ),
+        }
+
+        Ok(Some(Field {}))
+    }
+
+    pub fn encode(&self, ident: TokenStream) -> TokenStream {
+        quote! {
+            #ident.encode_raw(buf)
+        }
+    }
+
+    pub fn merge(&self, ident: TokenStream) -> TokenStream {
+        quote! {
+            #ident.merge_field(tag, wire_type, buf, ctx)
+        }
+    }
+
+    pub fn encoded_len(&self, ident: TokenStream) -> TokenStream {
+        quote! {
+            #ident.encoded_len()
+        }
+    }
+
+    pub fn clear(&self, ident: TokenStream) -> TokenStream {
+        quote! {
+            #ident.clear()
+        }
+    }
+}