Split fromfile support into its own submodule (#20807)

Background: Option @fromfile paths are intended to be relative
to the buildroot, not the cwd. This will require us to plumb
buildroot information through in several places.

This change is prep work for that change, and is straightforward
and technical, despite the diff stats: It just splits fromfile support
out into its own submodule. This will make the subsequent changes
easier to grok.

The new fromfile.rs and fromfile_tests.rs contain code moved
from parser.rs and parser_tests.rs, with no functionality changes.
The other diffs are just import changes.

Author: benjyw

src/rust/engine/options/src/args.rs

@@ -5,7 +5,8 @@ use std::env;
 
 use super::id::{is_valid_scope_name, NameTransform, OptionId, Scope};
 use super::{DictEdit, OptionsSource};
-use crate::parse::{expand, expand_to_dict, expand_to_list, ParseError, Parseable};
+use crate::fromfile::{expand, expand_to_dict, expand_to_list};
+use crate::parse::{ParseError, Parseable};
 use crate::ListEdit;
 use core::iter::once;
 use itertools::{chain, Itertools};

src/rust/engine/options/src/args_tests.rs

@@ -5,7 +5,7 @@ use core::fmt::Debug;
 use maplit::hashmap;
 
 use crate::args::Args;
-use crate::parse::test_util::write_fromfile;
+use crate::fromfile::test_util::write_fromfile;
 use crate::{option_id, DictEdit, DictEditAction, Val};
 use crate::{ListEdit, ListEditAction, OptionId, OptionsSource};
 

src/rust/engine/options/src/config.rs

@@ -10,9 +10,10 @@ use regex::Regex;
 use toml::value::Table;
 use toml::Value;
 
-use super::id::{NameTransform, OptionId};
-use super::parse::{expand, expand_to_dict, expand_to_list, Parseable};
 use super::{DictEdit, DictEditAction, ListEdit, ListEditAction, OptionsSource, Val};
+use crate::fromfile::{expand, expand_to_dict, expand_to_list};
+use crate::id::{NameTransform, OptionId};
+use crate::parse::Parseable;
 
 type InterpolationMap = HashMap<String, String>;
 

src/rust/engine/options/src/config_tests.rs

@@ -14,7 +14,7 @@ use crate::{
 };
 
 use crate::config::Config;
-use crate::parse::test_util::write_fromfile;
+use crate::fromfile::test_util::write_fromfile;
 use tempfile::TempDir;
 
 fn maybe_config(file_content: &str) -> Result<Config, String> {

src/rust/engine/options/src/env.rs

@@ -7,7 +7,8 @@ use std::ffi::OsString;
 
 use super::id::{NameTransform, OptionId, Scope};
 use super::{DictEdit, OptionsSource};
-use crate::parse::{expand, expand_to_dict, expand_to_list, Parseable};
+use crate::fromfile::{expand, expand_to_dict, expand_to_list};
+use crate::parse::Parseable;
 use crate::ListEdit;
 
 #[derive(Debug)]

src/rust/engine/options/src/env_tests.rs

@@ -2,7 +2,7 @@
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 use crate::env::Env;
-use crate::parse::test_util::write_fromfile;
+use crate::fromfile::test_util::write_fromfile;
 use crate::{option_id, DictEdit, DictEditAction};
 use crate::{ListEdit, ListEditAction, OptionId, OptionsSource, Val};
 use maplit::hashmap;

src/rust/engine/options/src/fromfile.rs

@@ -0,0 +1,139 @@
+// Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+use super::{DictEdit, DictEditAction, ListEdit, ListEditAction};
+
+use crate::parse::{mk_parse_err, parse_dict, ParseError, Parseable};
+use log::warn;
+use serde::de::Deserialize;
+use std::path::{Path, PathBuf};
+use std::{fs, io};
+
+// If the corresponding unexpanded value points to a @fromfile, then the
+// first component is the path to that file, and the second is the value from the file,
+// or None if the file doesn't exist and the @?fromfile syntax was used.
+//
+// Otherwise, the first component is None and the second is the original value.
+type ExpandedValue = (Option<PathBuf>, Option<String>);
+
+fn maybe_expand(value: String) -> Result<ExpandedValue, ParseError> {
+    if let Some(suffix) = value.strip_prefix('@') {
+        if suffix.starts_with('@') {
+            // @@ escapes the initial @.
+            Ok((None, Some(suffix.to_owned())))
+        } else {
+            match suffix.strip_prefix('?') {
+                Some(subsuffix) => {
+                    // @? means the path is allowed to not exist.
+                    let path = PathBuf::from(subsuffix);
+                    match fs::read_to_string(&path) {
+                        Ok(content) => Ok((Some(path), Some(content))),
+                        Err(err) if err.kind() == io::ErrorKind::NotFound => {
+                            warn!("Optional file config '{}' does not exist.", path.display());
+                            Ok((Some(path), None))
+                        }
+                        Err(err) => Err(mk_parse_err(err, &path)),
+                    }
+                }
+                _ => {
+                    let path = PathBuf::from(suffix);
+                    let content = fs::read_to_string(&path).map_err(|e| mk_parse_err(e, &path))?;
+                    Ok((Some(path), Some(content)))
+                }
+            }
+        }
+    } else {
+        Ok((None, Some(value)))
+    }
+}
+
+pub(crate) fn expand(value: String) -> Result<Option<String>, ParseError> {
+    let (_, expanded_value) = maybe_expand(value)?;
+    Ok(expanded_value)
+}
+
+#[derive(Debug)]
+enum FromfileType {
+    Json,
+    Yaml,
+    Unknown,
+}
+
+impl FromfileType {
+    fn detect(path: &Path) -> FromfileType {
+        if let Some(ext) = path.extension() {
+            if ext == "json" {
+                return FromfileType::Json;
+            } else if ext == "yml" || ext == "yaml" {
+                return FromfileType::Yaml;
+            };
+        }
+        FromfileType::Unknown
+    }
+}
+
+fn try_deserialize<'a, DE: Deserialize<'a>>(
+    value: &'a str,
+    path_opt: Option<PathBuf>,
+) -> Result<Option<DE>, ParseError> {
+    if let Some(path) = path_opt {
+        match FromfileType::detect(&path) {
+            FromfileType::Json => serde_json::from_str(value).map_err(|e| mk_parse_err(e, &path)),
+            FromfileType::Yaml => serde_yaml::from_str(value).map_err(|e| mk_parse_err(e, &path)),
+            _ => Ok(None),
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+pub(crate) fn expand_to_list<T: Parseable>(
+    value: String,
+) -> Result<Option<Vec<ListEdit<T>>>, ParseError> {
+    let (path_opt, value_opt) = maybe_expand(value)?;
+    if let Some(value) = value_opt {
+        if let Some(items) = try_deserialize(&value, path_opt)? {
+            Ok(Some(vec![ListEdit {
+                action: ListEditAction::Replace,
+                items,
+            }]))
+        } else {
+            T::parse_list(&value).map(Some)
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+pub(crate) fn expand_to_dict(value: String) -> Result<Option<Vec<DictEdit>>, ParseError> {
+    let (path_opt, value_opt) = maybe_expand(value)?;
+    if let Some(value) = value_opt {
+        if let Some(items) = try_deserialize(&value, path_opt)? {
+            Ok(Some(vec![DictEdit {
+                action: DictEditAction::Replace,
+                items,
+            }]))
+        } else {
+            parse_dict(&value).map(|x| Some(vec![x]))
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod test_util {
+    use std::fs::File;
+    use std::io::Write;
+    use std::path::PathBuf;
+    use tempfile::{tempdir, TempDir};
+
+    pub(crate) fn write_fromfile(filename: &str, content: &str) -> (TempDir, PathBuf) {
+        let tmpdir = tempdir().unwrap();
+        let fromfile_path = tmpdir.path().join(filename);
+        let mut fromfile = File::create(&fromfile_path).unwrap();
+        fromfile.write_all(content.as_bytes()).unwrap();
+        fromfile.flush().unwrap();
+        (tmpdir, fromfile_path)
+    }
+}