Add regex filtering to CLI

Author: htmlboss

src/CLIOptions.hpp

@@ -22,6 +22,7 @@ namespace tsm::cli {
         ("regen-indices", "Regenerate indices.", cxxopts::value<bool>())
         ("f,forecast", "Forecast dataset type.", cxxopts::value<bool>())
         ("h,historical", "Historical dataset type.", cxxopts::value<bool>())
+        ("r,regex", "Regex to apply to input directory.", cxxopts::value<std::string>())
         ("help", "Print help.")
         ;
 
@@ -49,13 +50,15 @@ struct [[nodiscard]] CLIOptions {
     explicit CLIOptions(const cxxopts::ParseResult& result) :   InputDir{ result["input-dir"].as<std::string>() },
                                                                 DatasetName{ result["dataset-name"].as<std::string>() },
                                                                 OutputDir{ result["output-dir"].as<std::string>() },
+                                                                RegexPattern{ result.count("regex") > 0 ? result["regex"].as<std::string>() : ".*" },
                                                                 RegenIndices{ result.count("regen-indices") > 0 },
                                                                 Forecast{ result.count("forecast") > 0 },
                                                                 Historical{ result.count("historical") > 0 } {}
 
     const std::string InputDir;
     const std::string DatasetName;
     const std::string OutputDir;
+    const std::string RegexPattern;
     const bool RegenIndices{ false };
     const bool Forecast{ false };
     const bool Historical{ false };

src/TimestampMapper.cpp

@@ -5,6 +5,7 @@
 #include <exception>
 #include <iostream>
 #include <fstream>
+#include <regex>
 
 namespace fs = std::filesystem;
 
@@ -14,11 +15,13 @@ namespace tsm {
 TimestampMapper::TimestampMapper(const std::filesystem::path& inputDir,
                                  const std::filesystem::path& outputDir,
                                  const std::string& datasetName,
+                                 const std::string& regexPattern,
                                  const ds::DATASET_TYPE datasetType,
                                  const bool regenIndices) : 
                                                         m_inputDir{ sanitizeDirectoryPath(inputDir) },
                                                         m_outputDir{ sanitizeDirectoryPath(outputDir) },
                                                         m_datasetName{ datasetName },
+                                                        m_regexPattern{ regexPattern },
                                                         m_datasetType{ datasetType },
                                                         m_regenIndices{ regenIndices },
                                                         m_filesToIndexPath{ m_inputDir / "files_to_index.txt" },
@@ -47,7 +50,7 @@ bool TimestampMapper::exec() {
     }
 
     std::cout << "Creating list of all .nc files in " << (m_indexFileExists ? m_filesToIndexPath : m_inputDir) << "..." << std::endl;
-    const auto& filePaths{ createFileList(m_indexFileExists ? m_filesToIndexPath : m_inputDir) };
+    const auto& filePaths{ createFileList(m_indexFileExists ? m_filesToIndexPath : m_inputDir, m_regexPattern) };
     if (filePaths.empty()) {
         std::cout << "No .nc files found." << "\nExiting..." << std::endl;
         return false;
@@ -90,7 +93,7 @@ bool TimestampMapper::createDirectory(const std::filesystem::path& path) const n
 }
 
 /***********************************************************************************/
-std::vector<fs::path> TimestampMapper::createFileList(const std::filesystem::path& inputDirOrIndexFile) const {
+std::vector<fs::path> TimestampMapper::createFileList(const std::filesystem::path& inputDirOrIndexFile, const std::string& regex) const {
     std::vector<fs::path> paths;
 
     // If file_to_index.txt exists, pull the file paths from there.
@@ -114,12 +117,24 @@ std::vector<fs::path> TimestampMapper::createFileList(const std::filesystem::pat
 
     using recursive_directory_iterator = std::filesystem::recursive_directory_iterator;
     const auto options{fs::directory_options::follow_directory_symlink};
+    
+    try {
+        const std::regex r(regex);
 
-    for (const auto& file : recursive_directory_iterator(inputDirOrIndexFile, options)) {
-        if (fs::path(file).extension() == ".nc") {
-            paths.emplace_back(file);
+        for (const auto& file : recursive_directory_iterator(inputDirOrIndexFile, options)) {
+            if (fs::path(file).extension() == ".nc" && std::regex_match(fs::path(file).string(), r)) {
+                paths.emplace_back(file);
+            }
         }
     }
+    catch(const std::regex_error& e) {
+        std::cerr << "Regex error: " << e.what() << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
+    catch(...) {
+        std::cerr << "Caught unknown exception." << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
 
     return paths;
 }

src/TimestampMapper.hpp

@@ -15,6 +15,7 @@ class TimestampMapper {
     TimestampMapper(const std::filesystem::path& inputDir,
                     const std::filesystem::path& outputDir,
                     const std::string& datasetName,
+                    const std::string& regexPattern,
                     const ds::DATASET_TYPE datasetType,
                     const bool regenIndices);
 
@@ -48,14 +49,15 @@ class TimestampMapper {
     [[nodiscard]] bool createDirectory(const std::filesystem::path& path) const noexcept;
     ///
     [[nodiscard]] std::vector<std::filesystem::path> 
-                                        createFileList(const std::filesystem::path& inputDirOrIndexFile) const;
+                                        createFileList(const std::filesystem::path& inputDirOrIndexFile, const std::string& regex) const;
 
     ///
     void deleteIndexFile();
 
     const std::filesystem::path m_inputDir;
     const std::filesystem::path m_outputDir;
     const std::string m_datasetName;
+    const std::string m_regexPattern;
     const ds::DATASET_TYPE m_datasetType;
     const bool m_regenIndices{ false };
 

src/main.cpp

@@ -28,6 +28,7 @@ int main(int argc, char** argv) {
     tsm::TimestampMapper mapper{opts.InputDir,
                                 opts.OutputDir,
                                 opts.DatasetName,
+                                opts.RegexPattern,
                                 opts.Forecast ? tsm::ds::DATASET_TYPE::FORECAST : tsm::ds::DATASET_TYPE::HISTORICAL, 
                                 opts.RegenIndices
                                 };