133 directory processing (#142)

* Changing Run to run directories, old run is now run_file.

* tests

* Directory processing with generators, adjusting old modules.

* testing and new data class

* Removing unused code, changing cli to match new dataclass

* error message change, using set for comparison

* renamed class to resultsdictionary.

* name change

* Using dict instead of custom dataclass

* doc string edits, writing in missing directories, removing directorynotfound error.

* ruff error

* Changes, and tutorial update

* errors, doc strings, additional tests.

* arg-type ignore

* variable assignment mistake

* Breaking up run_dir and run_file. Adding additional changes.

* Doc string, ruff error

* doc string

* doc strings

* Read me quick start update

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* final comment.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: frey-perez

README.md

@@ -50,12 +50,19 @@ pip install wristpy
 ## Quick start
 
 ### Using Wristpy through the command-line:
+#### Run single files:
 ```sh
 wristpy /input/file/path.gt3x -o /save/path/file_name.csv -c gradient
 ```
 
+#### Run entire directories:
+```sh
+wristpy /path/to/files/input_dir -o /path/to/files/output_dir -c gradient -O .csv
+```
+
 ### Using Wristpy through a python script or notebook:
 
+#### Running single files:
 ```Python
 
 from wristpy.core import orchestrator
@@ -72,13 +79,42 @@ results = orchestrator.run(
     calibrator='gradient',  # Choose between 'ggir', 'gradient', or 'none'
 )
 
-#Data availble in results object
+#Data available in results object
 enmo = results.enmo
 anglez = results.anglez
 physical_activity_levels = results.physical_activity_levels
 nonwear_array = results.nonwear_epoch
 sleep_windows = results.sleep_windows_epoch
 ```
+#### Running entire directories:
+```Python
+
+from wristpy.core import orchestrator
+
+# Define input file path and output location
+
+input_path = '/path/to/files/input_dir'
+output_path = '/path/to/files/output_dir'
+
+# Run the orchestrator
+# Specify the output file type, support for saving as .csv and .parquet
+results_dict = orchestrator.run(
+    input=input_path,
+    output=output_path,
+    calibrator='gradient',  # Choose between 'ggir', 'gradient', or 'none'
+    output_filetype = '.csv'
+)
+
+
+#Data available in dictionry of results.
+subject1 = results_dict['subject1']
+
+enmo = subject1.enmo
+anglez = subject1.anglez
+physical_activity_levels = subject1.physical_activity_levels
+nonwear_array = subject1.nonwear_epoch
+sleep_windows = subject1.sleep_windows_epoch
+```
 
 ## References
 1. van Hees, V.T., Sabia, S., Jones, S.E. et al. Estimating sleep parameters

docs/wristpy_tutorial.md

@@ -30,6 +30,20 @@ results = orchestrator.run(
 ```
 This runs the processing pipeline with all the default arguments, creates an output `.csv` file, and will create a `results` object that contains the various output metrics (namely, enmo, angle-z, physical activity values, non-wear detection, sleep detection).
 
+The orchestrator can also process entire directories. The call to the orchestrator remains largely the same but now output is expected to be a directory and the desired filetype for the saved files **must** be specified:
+
+```python
+from wristpy.core import orchestrator
+
+results = orchestrator.run(
+    input = '/path/to/input/dir',
+    output = '/path/to/output/dir',
+    output_filetype = ".csv"
+)
+```
+
+
+
 We can visualize some of the outputs within the `results` object, directly, with the following scripts:
 
 Plot the ENMO across the entire data set:

src/wristpy/__main__.py

@@ -1,16 +1,11 @@
 """Main function for wristpy."""
 
-from wristpy.core import cli, orchestrator
+from wristpy.core import cli
 
 
-def run_main() -> orchestrator.Results:
-    """Main entry point to wristpy.
-
-    Returns:
-        A Results object containing enmo, anglez, physical activity levels, nonwear
-        detection, and sleep detection.
-    """
-    return cli.main()
+def run_main() -> None:
+    """Main entry point to wristpy."""
+    cli.main()
 
 
 if __name__ == "__main__":

src/wristpy/core/cli.py

@@ -3,7 +3,7 @@
 import argparse
 import logging
 import pathlib
-from typing import List, Optional
+from typing import List, Optional, Tuple, cast
 
 from wristpy.core import config, orchestrator
 
@@ -35,6 +35,15 @@ def parse_arguments(args: Optional[List[str]] = None) -> argparse.Namespace:
         help="Path where data will be saved. Supports .csv and .parquet formats.",
     )
 
+    parser.add_argument(
+        "-O",
+        "--output_filetype",
+        type=str,
+        default=None,
+        help="Format for save files when processing directories. "
+        "Leave as None when processing single files.",
+    )
+
     parser.add_argument(
         "-c",
         "--calibrator",
@@ -83,7 +92,9 @@ def parse_arguments(args: Optional[List[str]] = None) -> argparse.Namespace:
     return parser.parse_args(args)
 
 
-def main(args: Optional[List[str]] = None) -> orchestrator.Results:
+def main(
+    args: Optional[List[str]] = None,
+) -> None:
     """Runs wristpy orchestrator with command line arguments.
 
     Args:
@@ -120,10 +131,12 @@ def main(args: Optional[List[str]] = None) -> orchestrator.Results:
 
     logger.debug("Running wristpy. arguments given: %s", arguments)
 
-    return orchestrator.run(
+    orchestrator.run(
         input=arguments.input,
         output=arguments.output,
-        thresholds=tuple(arguments.thresholds),
+        thresholds=cast(Tuple[float, float, float], tuple(arguments.thresholds)),
         calibrator=None if arguments.calibrator == "none" else arguments.calibrator,
         epoch_length=None if arguments.epoch_length == 0 else arguments.epoch_length,
+        verbosity=log_level,
+        output_filetype=arguments.output_filetype,
     )

src/wristpy/core/exceptions.py

@@ -40,9 +40,3 @@ class InvalidFileTypeError(LoggedException):
     """Wristpy did not expect this file extension."""
 
     pass
-
-
-class DirectoryNotFoundError(LoggedException):
-    """Output save path not found."""
-
-    pass

src/wristpy/core/models.py

@@ -1,11 +1,19 @@
 """Internal data model."""
 
+import pathlib
 from typing import Optional
 
 import numpy as np
 import polars as pl
+import pydantic
 from pydantic import BaseModel, field_validator
 
+from wristpy.core import config, exceptions
+
+VALID_FILE_TYPES = (".csv", ".parquet")
+
+logger = config.get_logger()
+
 
 class Measurement(BaseModel):
     """A single measurement of a sensor and its corresponding time."""
@@ -125,3 +133,59 @@ def validate_acceleration(cls, v: Measurement) -> Measurement:
         if v.measurements.ndim != 2 or v.measurements.shape[1] != 3:
             raise ValueError("acceleration must be a 2D array with 3 columns")
         return v
+
+
+class OrchestratorResults(pydantic.BaseModel):
+    """Dataclass containing results of orchestrator.run()."""
+
+    enmo: Measurement
+    anglez: Measurement
+    physical_activity_levels: Measurement
+    nonwear_epoch: Measurement
+    sleep_windows_epoch: Measurement
+
+    def save_results(self, output: pathlib.Path) -> None:
+        """Convert to polars and save the dataframe as a csv or parquet file.
+
+        Args:
+            output: The path and file name of the data to be saved. as either a csv or
+                parquet files.
+
+        """
+        logger.debug("Saving results.")
+        self.validate_output(output=output)
+        output.parent.mkdir(parents=True, exist_ok=True)
+
+        results_dataframe = pl.DataFrame(
+            {"time": self.enmo.time}
+            | {name: value.measurements for name, value in self}
+        )
+
+        if output.suffix == ".csv":
+            results_dataframe.write_csv(output, separator=",")
+        elif output.suffix == ".parquet":
+            results_dataframe.write_parquet(output)
+        else:
+            raise exceptions.InvalidFileTypeError(
+                f"File type must be one of {VALID_FILE_TYPES}"
+            )
+
+        logger.debug("results saved in: %s", output)
+
+    @classmethod
+    def validate_output(cls, output: pathlib.Path) -> None:
+        """Validates that the output path exists and is a valid format.
+
+        Args:
+            output: the name of the file to be saved, and the directory it will
+                be saved in. Must be a .csv or .parquet file.
+
+        Raises:
+            InvalidFileTypeError:If the output file path ends with any extension other
+                    than csv or parquet.
+        """
+        if output.suffix not in VALID_FILE_TYPES:
+            raise exceptions.InvalidFileTypeError(
+                f"The extension: {output.suffix} is not supported."
+                "Please save the file as .csv or .parquet",
+            )