Generate option value derivations in the rust parser. (#20535)

We use these to show the source of option values in help output.

The derivations can be a lot of data, so we only produce them
if asked to.

Note that the derivations currently only show the source type
(args, env, config, default) but not, for example, which config
file a value came from. A future change may add this,  but it
will require some changes to the Source enum which are out
of scope for this PR. 

Also: use maplit for easy hashmap literals in the test.

Author: benjyw

src/rust/engine/Cargo.lock

@@ -40,6 +40,7 @@ async fn test_client_fingerprint_mismatch() {
         Env::new(HashMap::new()),
         None,
         true,
+        false,
     )
     .unwrap();
     let error = pantsd::find_pantsd(&build_root, &options_parser)

src/rust/engine/client/src/client_tests.rs

@@ -32,7 +32,7 @@ async fn execute(start: SystemTime) -> Result<i32, String> {
     let (env, dropped) = Env::capture_lossy();
     let env_items = (&env).into();
     let argv = env::args().collect::<Vec<_>>();
-    let options_parser = OptionParser::new(Args::argv(), env, None, true)?;
+    let options_parser = OptionParser::new(Args::argv(), env, None, true, false)?;
 
     let use_pantsd = options_parser.parse_bool(&option_id!("pantsd"), true)?;
     if !use_pantsd.value {

src/rust/engine/client/src/main.rs

@@ -8,6 +8,7 @@ publish = false
 [dependencies]
 lazy_static = { workspace = true }
 log = { workspace = true }
+maplit = { workspace = true }
 peg = { workspace = true }
 shellexpand = { workspace = true }
 toml = { workspace = true }

src/rust/engine/options/Cargo.toml

@@ -33,7 +33,6 @@ mod types;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::Debug;
 use std::hash::Hash;
-use std::ops::Deref;
 use std::os::unix::ffi::OsStrExt;
 use std::path;
 use std::path::Path;
@@ -56,7 +55,7 @@ pub use types::OptionType;
 // We only use this for parsing values in dicts, as in other cases we know that the type must
 // be some scalar or string, or a uniform list of one type of scalar or string, so we can
 // parse as such.
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Val {
     Bool(bool),
     Int(i64),
@@ -67,26 +66,26 @@ pub enum Val {
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub(crate) enum ListEditAction {
+pub enum ListEditAction {
     Replace,
     Add,
     Remove,
 }
 
-#[derive(Debug, Eq, PartialEq)]
-pub(crate) struct ListEdit<T> {
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ListEdit<T> {
     pub action: ListEditAction,
     pub items: Vec<T>,
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub(crate) enum DictEditAction {
+pub enum DictEditAction {
     Replace,
     Add,
 }
 
-#[derive(Debug, PartialEq)]
-pub(crate) struct DictEdit {
+#[derive(Clone, Debug, PartialEq)]
+pub struct DictEdit {
     pub action: DictEditAction,
     pub items: HashMap<String, Val>,
 }
@@ -200,20 +199,28 @@ pub enum Source {
 
 #[derive(Debug)]
 pub struct OptionValue<T> {
+    pub derivation: Option<Vec<(Source, T)>>,
+    // Scalar options are always set from a single source, so we provide that
+    // here, as it can be useful in user-facing messages.
     pub source: Source,
     pub value: T,
 }
 
-impl<T> Deref for OptionValue<T> {
-    type Target = T;
+#[derive(Debug)]
+pub struct ListOptionValue<T> {
+    pub derivation: Option<Vec<(Source, Vec<ListEdit<T>>)>>,
+    pub value: Vec<T>,
+}
 
-    fn deref(&self) -> &Self::Target {
-        &self.value
-    }
+#[derive(Debug)]
+pub struct DictOptionValue {
+    pub derivation: Option<Vec<(Source, Vec<DictEdit>)>>,
+    pub value: HashMap<String, Val>,
 }
 
 pub struct OptionParser {
     sources: BTreeMap<Source, Rc<dyn OptionsSource>>,
+    include_derivation: bool,
 }
 
 impl OptionParser {
@@ -224,6 +231,7 @@ impl OptionParser {
         env: Env,
         config_paths: Option<Vec<&str>>,
         allow_pantsrc: bool,
+        include_derivation: bool,
     ) -> Result<OptionParser, String> {
         let buildroot = BuildRoot::find()?;
         let buildroot_string = String::from_utf8(buildroot.as_os_str().as_bytes().to_vec())
@@ -246,6 +254,7 @@ impl OptionParser {
         sources.insert(Source::Flag, Rc::new(args));
         let mut parser = OptionParser {
             sources: sources.clone(),
+            include_derivation: false,
         };
 
         fn path_join(a: &str, b: &str) -> String {
@@ -256,10 +265,12 @@ impl OptionParser {
             Some(paths) => paths.iter().map(|s| s.to_string()).collect(),
             None => {
                 let default_config_path = path_join(&buildroot_string, "pants.toml");
-                parser.parse_string_list(
-                    &option_id!("pants", "config", "files"),
-                    &[&default_config_path],
-                )?
+                parser
+                    .parse_string_list(
+                        &option_id!("pants", "config", "files"),
+                        &[&default_config_path],
+                    )?
+                    .value
             }
         };
 
@@ -285,17 +296,21 @@ impl OptionParser {
         sources.insert(Source::Config, Rc::new(config.clone()));
         parser = OptionParser {
             sources: sources.clone(),
+            include_derivation: false,
         };
 
-        if allow_pantsrc && *parser.parse_bool(&option_id!("pantsrc"), true)? {
-            for rcfile in parser.parse_string_list(
-                &option_id!("pantsrc", "files"),
-                &[
-                    "/etc/pantsrc",
-                    shellexpand::tilde("~/.pants.rc").as_ref(),
-                    ".pants.rc",
-                ],
-            )? {
+        if allow_pantsrc && parser.parse_bool(&option_id!("pantsrc"), true)?.value {
+            for rcfile in parser
+                .parse_string_list(
+                    &option_id!("pantsrc", "files"),
+                    &[
+                        "/etc/pantsrc",
+                        shellexpand::tilde("~/.pants.rc").as_ref(),
+                        ".pants.rc",
+                    ],
+                )?
+                .value
+            {
                 let rcfile_path = Path::new(&rcfile);
                 if rcfile_path.exists() {
                     let rc_config = Config::parse(&[rcfile_path], &seed_values)?;
@@ -304,7 +319,10 @@ impl OptionParser {
             }
         }
         sources.insert(Source::Config, Rc::new(config));
-        Ok(OptionParser { sources })
+        Ok(OptionParser {
+            sources,
+            include_derivation,
+        })
     }
 
     #[allow(clippy::type_complexity)]
@@ -314,15 +332,27 @@ impl OptionParser {
         default: &T,
         getter: fn(&Rc<dyn OptionsSource>, &OptionId) -> Result<Option<T::Owned>, String>,
     ) -> Result<OptionValue<T::Owned>, String> {
+        let mut derivation = None;
+        if self.include_derivation {
+            let mut derivations = vec![(Source::Default, default.to_owned())];
+            for (source_type, source) in self.sources.iter().rev() {
+                if let Some(val) = getter(source, id)? {
+                    derivations.push((*source_type, val));
+                }
+            }
+            derivation = Some(derivations);
+        }
         for (source_type, source) in self.sources.iter() {
             if let Some(value) = getter(source, id)? {
                 return Ok(OptionValue {
+                    derivation,
                     source: *source_type,
                     value,
                 });
             }
         }
         Ok(OptionValue {
+            derivation,
             source: Source::Default,
             value: default.to_owned(),
         })
@@ -349,14 +379,32 @@ impl OptionParser {
     }
 
     #[allow(clippy::type_complexity)]
-    fn parse_list<T>(
+    fn parse_list<T: Clone>(
         &self,
         id: &OptionId,
         default: Vec<T>,
         getter: fn(&Rc<dyn OptionsSource>, &OptionId) -> Result<Option<Vec<ListEdit<T>>>, String>,
         remover: fn(&mut Vec<T>, &Vec<T>),
-    ) -> Result<Vec<T>, String> {
+    ) -> Result<ListOptionValue<T>, String> {
         let mut list = default;
+        let mut derivation = None;
+        if self.include_derivation {
+            let mut derivations = vec![(
+                Source::Default,
+                vec![ListEdit {
+                    action: ListEditAction::Replace,
+                    items: list.clone(),
+                }],
+            )];
+            for (source_type, source) in self.sources.iter().rev() {
+                if let Some(list_edits) = getter(source, id)? {
+                    if !list_edits.is_empty() {
+                        derivations.push((*source_type, list_edits));
+                    }
+                }
+            }
+            derivation = Some(derivations);
+        }
         for (_source_type, source) in self.sources.iter().rev() {
             if let Some(list_edits) = getter(source, id)? {
                 for list_edit in list_edits {
@@ -368,7 +416,10 @@ impl OptionParser {
                 }
             }
         }
-        Ok(list)
+        Ok(ListOptionValue {
+            derivation,
+            value: list,
+        })
     }
 
     // For Eq+Hash types we can use a HashSet when computing removals, which will be avg O(N+M).
@@ -378,28 +429,40 @@ impl OptionParser {
     // However this is still more than fast enough, and inoculates us against a very unlikely
     // pathological case of a very large removal set.
     #[allow(clippy::type_complexity)]
-    fn parse_list_hashable<T: Eq + Hash>(
+    fn parse_list_hashable<T: Clone + Eq + Hash>(
         &self,
         id: &OptionId,
         default: Vec<T>,
         getter: fn(&Rc<dyn OptionsSource>, &OptionId) -> Result<Option<Vec<ListEdit<T>>>, String>,
-    ) -> Result<Vec<T>, String> {
+    ) -> Result<ListOptionValue<T>, String> {
         self.parse_list(id, default, getter, |list, remove| {
             let to_remove = remove.iter().collect::<HashSet<_>>();
             list.retain(|item| !to_remove.contains(item));
         })
     }
 
-    pub fn parse_bool_list(&self, id: &OptionId, default: &[bool]) -> Result<Vec<bool>, String> {
+    pub fn parse_bool_list(
+        &self,
+        id: &OptionId,
+        default: &[bool],
+    ) -> Result<ListOptionValue<bool>, String> {
         self.parse_list_hashable(id, default.to_vec(), |source, id| source.get_bool_list(id))
     }
 
-    pub fn parse_int_list(&self, id: &OptionId, default: &[i64]) -> Result<Vec<i64>, String> {
+    pub fn parse_int_list(
+        &self,
+        id: &OptionId,
+        default: &[i64],
+    ) -> Result<ListOptionValue<i64>, String> {
         self.parse_list_hashable(id, default.to_vec(), |source, id| source.get_int_list(id))
     }
 
     // Floats are not Eq or Hash, so we fall back to the brute-force O(N*M) lookups.
-    pub fn parse_float_list(&self, id: &OptionId, default: &[f64]) -> Result<Vec<f64>, String> {
+    pub fn parse_float_list(
+        &self,
+        id: &OptionId,
+        default: &[f64],
+    ) -> Result<ListOptionValue<f64>, String> {
         self.parse_list(
             id,
             default.to_vec(),
@@ -414,7 +477,7 @@ impl OptionParser {
         &self,
         id: &OptionId,
         default: &[&str],
-    ) -> Result<Vec<String>, String> {
+    ) -> Result<ListOptionValue<String>, String> {
         self.parse_list_hashable::<String>(
             id,
             default.iter().map(|s| s.to_string()).collect(),
@@ -426,8 +489,24 @@ impl OptionParser {
         &self,
         id: &OptionId,
         default: HashMap<String, Val>,
-    ) -> Result<HashMap<String, Val>, String> {
+    ) -> Result<DictOptionValue, String> {
         let mut dict = default;
+        let mut derivation = None;
+        if self.include_derivation {
+            let mut derivations = vec![(
+                Source::Default,
+                vec![DictEdit {
+                    action: DictEditAction::Replace,
+                    items: dict.clone(),
+                }],
+            )];
+            for (source_type, source) in self.sources.iter().rev() {
+                if let Some(dict_edits) = source.get_dict(id)? {
+                    derivations.push((*source_type, dict_edits));
+                }
+            }
+            derivation = Some(derivations);
+        }
         for (_, source) in self.sources.iter().rev() {
             if let Some(dict_edits) = source.get_dict(id)? {
                 for dict_edit in dict_edits {
@@ -438,7 +517,10 @@ impl OptionParser {
                 }
             }
         }
-        Ok(dict)
+        Ok(DictOptionValue {
+            derivation,
+            value: dict,
+        })
     }
 }