--collect-as and --name-inputs (#162)

- feature: New flags `--collect-as` and `--name-inputs` have been implemented to aggregate data from arbitrary numbers of inputs (e.g.using shell wildcard syntax).

Author: gmorpheme

docs/command-line.md

@@ -145,12 +145,107 @@ When several inputs are listed, names from earlier inputs become
 available to later inputs, but the content that will be rendered is
 that of the final input.
 
-The common use case is a final input which contains logic to inspect
-or process data provided by the previous inputs (potentially coming in
-from previous processing on stdin).
+So for instance:
 
-If you want to render contents of earlier inputs verbatim, you need a
-named input to provide a name for that content which you can then use.
+a.eu
+```eu
+x: 4
+y: 8
+```
+
+b.eu
+
+```eu
+z: x + y
+```
+
+```sh
+eu a.eu b.eu
+```
+
+will output
+
+```yaml
+z: 12
+```
+
+The common use cases are:
+- a final input containing logic to inspect or process data
+  provided by previous inputs
+- a final input which uses functions defined in earlier inputs to
+  process data provided in previous inputs
+
+If you want to __render_ contents of earlier inputs, you need a named
+input to provide a name for that content which you can then use.
+
+For instance:
+
+```sh
+eu r=a.eu b.eu -e r
+```
+
+will render:
+
+```yaml
+x: 4
+y: 8
+```
+
+#### `--collect-as` and `--name-inputs`
+
+Occasionally it is useful to aggregate data from an arbitrary number
+of sources files, typically specified by shell wildcards. To refer to
+this data we need to introduce a name for the collection of data.
+
+This is what the command line switch `--collect-as` / `-C` is for.
+
+```sh
+eu --collect-as inputs *.eu
+```
+
+...will render:
+
+```yaml
+inputs:
+  - x: 4
+	y: 8
+  - z: 12
+```
+
+It is common to use `-e` to select an item to render:
+
+```sh
+eu -C *.eu -e 'inputs head'
+```
+
+...renders:
+
+```yaml
+x: 4
+y: 8
+```
+
+If you are likely to need to refer to inputs by name, you can add
+`--name-inputs` / `-N` to pass inputs as a block instead of a list:
+
+```sh
+eu --collect-as inputs *.eu
+```
+
+...renders:
+
+```yaml
+inputs:
+  a.eu:
+	x: 4
+	y: 8
+  b.eu:
+	z: 12
+```
+
+This makes it possible to easier to invoke specific functions from
+named inputs although you will need single-quote name syntax to use
+the generated names which contain '.'s.
 
 ## Outputs
 

src/core/doc.rs

@@ -12,3 +12,16 @@ pub struct DeclarationDocumentation {
     /// Documentation metadata
     pub doc: String,
 }
+
+impl DeclarationDocumentation {
+    /// Construct a documentation item one level deeper under a named
+    /// root
+    pub fn under(&self, root: &str) -> Self {
+        let mut path = vec![root.to_string()];
+        path.extend(self.path.clone());
+        Self {
+            path,
+            ..self.clone()
+        }
+    }
+}

src/core/target.rs

@@ -66,4 +66,14 @@ impl Target {
     pub fn path(&self) -> &Vec<String> {
         &self.path
     }
+
+    /// Return a version of the target where the path is
+    pub fn under(&self, root: &str) -> Self {
+        let mut path = vec![root.to_string()];
+        path.extend(self.path.clone());
+        Self {
+            path,
+            ..self.clone()
+        }
+    }
 }

src/core/unit.rs

@@ -42,6 +42,45 @@ impl TranslationUnit {
         })
     }
 
+    /// Combine the units as a unit containing a single list
+    ///
+    /// No targets or
+    pub fn listify<'a, I>(units: I) -> Result<Self, CoreError>
+    where
+        I: Iterator<Item = &'a Self>,
+    {
+        Ok(TranslationUnit {
+            expr: acore::list(units.map(|u| u.expr.clone()).collect()),
+            ..Default::default()
+        })
+    }
+
+    /// Combine the units as a unit containing a single block with
+    /// each units keys derived from existing name or generated name
+    /// if required
+    pub fn blockify<'a, I, S>(keys: &'a [S], units: I) -> Result<Self, CoreError>
+    where
+        I: Iterator<Item = &'a Self>,
+        S: AsRef<str>,
+    {
+        let mut targets = HashSet::new();
+        let mut docs = vec![];
+        let mut entries = vec![];
+
+        for (k, u) in keys.iter().zip(units) {
+            let key = k.as_ref();
+            targets.extend(u.targets.iter().map(|t| t.under(key)));
+            docs.extend(u.docs.iter().map(|d| d.under(key)));
+            entries.push((key.to_string(), u.expr.clone()));
+        }
+
+        Ok(TranslationUnit {
+            expr: acore::block(entries),
+            targets,
+            docs,
+        })
+    }
+
     /// Return a new unit which merges the expression and targets etc.
     /// of the specified units.
     pub fn merge<I>(units: I) -> Result<Self, CoreError>

src/driver/options.rs

@@ -59,6 +59,12 @@ pub struct EucalyptOptions {
     #[structopt(short, long)]
     evaluate: Option<String>,
 
+    #[structopt(short = "c", long)]
+    collect_as: Option<String>,
+
+    #[structopt(short = "N", long)]
+    name_inputs: bool,
+
     /// Turn on debug features
     #[structopt(short, long)]
     debug: bool,
@@ -92,8 +98,16 @@ pub struct EucalyptOptions {
     #[structopt(flatten)]
     stg_settings: StgSettings,
 
-    /// Source code / data inputs (in order)
-    inputs: Vec<Input>,
+    /// Explicit source code / data inputs (in order)
+    explicit_inputs: Vec<Input>,
+
+    /// Prologue inputs (inputs added prior to explicit inputs)
+    #[structopt(skip)]
+    prologue_inputs: Vec<Input>,
+
+    /// Epilogue inputs (inputs added after explicit inputs)
+    #[structopt(skip)]
+    epilogue_inputs: Vec<Input>,
 }
 
 impl EucalyptOptions {
@@ -114,8 +128,8 @@ impl EucalyptOptions {
         self
     }
 
-    pub fn with_inputs(mut self, inputs: Vec<Input>) -> Self {
-        self.inputs = inputs;
+    pub fn with_explicit_inputs(mut self, inputs: Vec<Input>) -> Self {
+        self.explicit_inputs = inputs;
         self
     }
 
@@ -193,7 +207,19 @@ impl EucalyptOptions {
 
         // inputs
         explanation.push_str("Inputs:\n");
-        for i in &self.inputs {
+        for i in &self.prologue_inputs {
+            explanation.push_str(&format!(" • {}\n", i));
+        }
+        if let Some(name) = self.collection() {
+            explanation.push_str(&format!(" • {}=\n", name));
+        }
+        for i in &self.explicit_inputs {
+            if self.collection().is_some() {
+                explanation.push_str("  ");
+            }
+            explanation.push_str(&format!(" • {}\n", i));
+        }
+        for i in &self.epilogue_inputs {
             explanation.push_str(&format!(" • {}\n", i));
         }
         explanation.push('\n');
@@ -316,6 +342,14 @@ impl EucalyptOptions {
         self.evaluate.as_ref()
     }
 
+    pub fn collection(&self) -> Option<&String> {
+        self.collect_as.as_ref()
+    }
+
+    pub fn name_inputs(&self) -> bool {
+        self.name_inputs
+    }
+
     pub fn explain(&self) -> bool {
         self.command.explain
     }
@@ -372,8 +406,24 @@ impl EucalyptOptions {
         self.debug
     }
 
-    pub fn inputs(&self) -> &[Input] {
-        &self.inputs
+    pub fn prologue_inputs(&self) -> &[Input] {
+        &self.prologue_inputs
+    }
+
+    pub fn explicit_inputs(&self) -> &[Input] {
+        &self.explicit_inputs
+    }
+
+    pub fn epilogue_inputs(&self) -> &[Input] {
+        &self.epilogue_inputs
+    }
+
+    pub fn inputs(&self) -> Vec<Input> {
+        let mut inputs = vec![];
+        inputs.extend_from_slice(&self.prologue_inputs.as_slice());
+        inputs.extend_from_slice(&self.explicit_inputs.as_slice());
+        inputs.extend_from_slice(&self.epilogue_inputs.as_slice());
+        inputs
     }
 
     pub fn lib_path(&self) -> &[PathBuf] {
@@ -406,8 +456,8 @@ impl EucalyptOptions {
 
     /// Prepend an input if it is not already in the input list
     fn prepend_input(&mut self, input: Input) {
-        if !self.inputs.iter().any(|i| i == &input) {
-            self.inputs.insert(0, input);
+        if !self.explicit_inputs.iter().any(|i| i == &input) {
+            self.prologue_inputs.insert(0, input);
         }
     }
 
@@ -445,7 +495,7 @@ impl EucalyptOptions {
         self.lib_path.insert(0, std::env::current_dir()?);
 
         // For pipes, default json stdin
-        if self.inputs.is_empty() && !atty::is(Stream::Stdin) {
+        if self.explicit_inputs.is_empty() && !atty::is(Stream::Stdin) {
             self.prepend_input(Input::from_str("-").unwrap());
         }
 
@@ -484,7 +534,7 @@ impl EucalyptOptions {
 
         // Add CLI evaluand as final input if it exists
         if self.evaluate.is_some() {
-            self.inputs.push(Input::from(Locator::Cli(
+            self.epilogue_inputs.push(Input::from(Locator::Cli(
                 self.evaluate.as_ref().unwrap().to_string(),
             )));
         }

src/driver/prepare.rs

@@ -26,10 +26,12 @@ pub fn prepare(
     loader: &mut SourceLoader,
     stats: &mut Timings,
 ) -> Result<Command, EucalyptError> {
+    let inputs = opt.inputs();
+
     {
         let t = Instant::now();
 
-        for i in opt.inputs() {
+        for i in &inputs {
             loader.load(i)?;
         }
 
@@ -49,18 +51,28 @@ pub fn prepare(
     {
         let t = Instant::now();
 
-        for i in opt.inputs() {
+        for i in &inputs {
             loader.translate(i)?;
         }
 
         stats.record("translate", t.elapsed());
     }
 
-    // Merge into a single core - before or after cooking?
+    // Optionally collect explicit inputs together
     {
         let t = Instant::now();
 
-        loader.merge_units(opt.inputs())?;
+        if let Some(collection_name) = opt.collection() {
+            loader.collect_and_merge_units(
+                &collection_name,
+                opt.name_inputs(),
+                opt.prologue_inputs(),
+                opt.explicit_inputs(),
+                opt.epilogue_inputs(),
+            )?;
+        } else {
+            loader.merge_units(&inputs)?;
+        }
 
         stats.record("merge", t.elapsed());
     }
@@ -86,7 +98,7 @@ pub fn prepare(
         }
 
         println!("\nFrom inputs\n");
-        for i in opt.inputs() {
+        for i in &inputs {
             println!("  - {}", i);
         }
         return Ok(Command::Exit);