Add .toml files to option "fromfile" support. (#23196)

As a reminder, the "fromfile" feature is where an
option value `@path/to/file` is read from that file.

If the file is .json or .yaml, we deserialize its content
to a list or dict, as needed.

This change adds support for .toml.

A top-level TOML entity must be a table (unlike
JSON/YAML which can have a top-level list).
So we only support this for dict-valued options.

Author: benjyw

docs/docs/using-pants/key-concepts/options.mdx

@@ -49,7 +49,7 @@ Global options are set using the environment variable `PANTS_{OPTION_NAME}`:
 PANTS_LEVEL=debug pants ...
 ```
 
-Subsystem options are set using the environment variable  
+Subsystem options are set using the environment variable
 `PANTS_{SCOPE}_{OPTION_NAME}`:
 
 ```bash
@@ -330,7 +330,7 @@ values directly in `pants.toml`, you can set the value of any option to the stri
 `@relative/path/from/repo/root/to/file` (note the leading `@`), and the value will be read
 from that file.
 
-If the file name ends with `.json` or `.yaml` then the file will be parsed as the relevant
+If the file name ends with `.json`, `.yaml` or `.toml` then the file will be parsed as the relevant
 format, which is useful for list- and dict-valued options.
 
 Otherwise, the file is parsed as a literal as described above for each option type.

docs/notes/2.32.x.md

@@ -22,6 +22,8 @@ The plugin API's `Get()` and `MultiGet()` constructs, deprecated in 2.30, are no
 
 The Pants [Contribution Overview](https://www.pantsbuild.org/2.32/docs/contributions) now contains guidance on LLM use.
 
+Dict-valued options can now [read their values](https://www.pantsbuild.org/2.32/docs/using-pants/key-concepts/options#reading-individual-option-values-from-files) from `.toml` files.
+
 #### Internal Python Upgrade
 
 The version of Python used by Pants itself has been updated to [3.14](https://docs.python.org/3/whatsnew/3.14.html). To support this, the [Pants Launcher Binary](https://www.pantsbuild.org/blog/2023/02/23/the-pants-launcher-binary-a-much-simpler-way-to-install-and-run-pants) (also known as [`scie-pants`](https://github.com/pantsbuild/scie-pants/)) now has a minimum version of `0.13.0`. To update to the latest launcher binary, either:

src/rust/Cargo.lock

@@ -161,6 +161,7 @@ hyper-util = "0.1"
 ignore = "0.4.25"
 indexmap = { version = "2.13.0", features = ["std", "serde"] }
 indicatif = "0.18.4"
+indoc = "2.0.7"
 internment = "0.6"
 itertools = "0.14"
 libc = "0.2.182"

src/rust/Cargo.toml

@@ -23,6 +23,7 @@ whoami = { workspace = true }
 [dev-dependencies]
 maplit = { workspace = true }
 tempfile = { workspace = true }
+indoc = { workspace = true }
 
 [lints]
 workspace = true

src/rust/options/Cargo.toml

@@ -5,7 +5,7 @@ use super::{BuildRoot, DictEdit, DictEditAction, ListEdit, ListEditAction};
 
 use crate::parse::{ParseError, Parseable, mk_parse_err, parse_dict};
 use log::warn;
-use serde::de::Deserialize;
+use serde::de::DeserializeOwned;
 use sha2::{Digest, Sha256};
 use std::os::unix::ffi::OsStrExt;
 use std::path::{Path, PathBuf};
@@ -22,6 +22,7 @@ type ExpandedValue = (Option<PathBuf>, Option<String>);
 enum FromfileType {
     Json,
     Yaml,
+    Toml,
     Unknown,
 }
 
@@ -32,20 +33,23 @@ impl FromfileType {
                 return FromfileType::Json;
             } else if ext == "yml" || ext == "yaml" {
                 return FromfileType::Yaml;
+            } else if ext == "toml" {
+                return FromfileType::Toml;
             };
         }
         FromfileType::Unknown
     }
 }
 
-fn try_deserialize<'a, DE: Deserialize<'a>>(
-    value: &'a str,
+fn try_deserialize<DE: DeserializeOwned>(
+    value: &str,
     path_opt: Option<PathBuf>,
 ) -> Result<Option<DE>, ParseError> {
     if let Some(path) = path_opt {
         match FromfileType::detect(&path) {
             FromfileType::Json => serde_json::from_str(value).map_err(|e| mk_parse_err(e, &path)),
             FromfileType::Yaml => serde_yaml::from_str(value).map_err(|e| mk_parse_err(e, &path)),
+            FromfileType::Toml => toml::from_str(value).map_err(|e| mk_parse_err(e, &path)),
             _ => Ok(None),
         }
     } else {

src/rust/options/src/fromfile.rs

@@ -5,6 +5,7 @@ use crate::fromfile::test_util::write_fromfile;
 use crate::fromfile::*;
 use crate::parse::{ParseError, Parseable};
 use crate::{BuildRoot, DictEdit, DictEditAction, ListEdit, ListEditAction, Val};
+use indoc::indoc;
 use maplit::hashmap;
 use std::collections::HashMap;
 use std::fmt::Debug;
@@ -13,7 +14,7 @@ macro_rules! check_err {
     ($res:expr, $expected_suffix:expr $(,)?) => {
         let actual_msg = $res.unwrap_err().render("XXX");
         assert!(
-            actual_msg.ends_with($expected_suffix),
+            actual_msg.trim_end().ends_with($expected_suffix.trim_end()),
             "Error message does not have expected suffix:\n{actual_msg}\nvs\n{:>width$}",
             $expected_suffix,
             width = actual_msg.len(),
@@ -271,19 +272,32 @@ fn test_expand_fromfile_to_dict() {
         "fromfile.json",
     );
     do_test(
-        r#"
+        indoc! {"
         FOO:
           BAR: 3.14
           BAZ:
             QUX: true
             QUUX:
               - 1
               - 2
-        "#,
+        "},
         &complex_obj,
         "fromfile.yaml",
     );
 
+    do_test(
+        indoc! {"
+        [FOO]
+        BAR = 3.14
+        
+        [FOO.BAZ]
+        QUX= true
+        QUUX = [1, 2]
+        "},
+        &complex_obj,
+        "fromfile.toml",
+    );
+
     check_err!(
         expand_fromfile("THIS IS NOT JSON", "@", "invalid.json"),
         "expected value at line 1 column 1",
@@ -304,6 +318,17 @@ fn test_expand_fromfile_to_dict() {
         "invalid type: sequence, expected a map",
     );
 
+    check_err!(
+        expand_fromfile("THIS IS NOT TOML", "@", "invalid.toml"),
+        indoc! {"
+        TOML parse error at line 1, column 6
+          |
+        1 | THIS IS NOT TOML
+          |      ^
+        expected `.`, `=`
+        "},
+    );
+
     check_err!(
         expand_to_dict("@/does/not/exist".to_string()),
         "Problem reading /does/not/exist for XXX: No such file or directory (os error 2)",