diff --git a/src/vt/collective/collective_ops.cc b/src/vt/collective/collective_ops.cc index 09d897abc5..b8c9e73380 100644 --- a/src/vt/collective/collective_ops.cc +++ b/src/vt/collective/collective_ops.cc @@ -46,6 +46,7 @@ #include "vt/runtime/runtime.h" #include "vt/scheduler/scheduler.h" #include "vt/runtime/runtime_inst.h" +#include "vt/configs/arguments/args.h" #include #include @@ -216,7 +217,7 @@ void printOverwrittens( template RuntimePtrType CollectiveAnyOps::initialize( int& argc, char**& argv, bool is_interop, MPI_Comm* comm, - arguments::AppConfig const* appConfig + arguments::AppConfig const* appConfig, bool print_startup_banner ) { using vt::runtime::RuntimeInst; using vt::runtime::Runtime; @@ -235,7 +236,7 @@ RuntimePtrType CollectiveAnyOps::initialize( ::vt::rt = rt_ptr; curRT = rt_ptr; } - RuntimeInst::rt->initialize(); + RuntimeInst::rt->initialize(false, print_startup_banner); // If appConfig is not nullptr, compare CLI arguments with user-defined ones, // and report overwritten ones. @@ -246,6 +247,33 @@ RuntimePtrType CollectiveAnyOps::initialize( return runtime::makeRuntimePtr(rt_ptr); } +template +/*static*/ RuntimePtrType CollectiveAnyOps::initializePreconfigured( + std::unique_ptr startup_config, MPI_Comm* comm, + arguments::AppConfig const* app_config, bool print_startup_banner +) { + using vt::runtime::RuntimeInst; + using vt::runtime::Runtime; + using vt::runtime::eRuntimeInstance; + + MPI_Comm resolved_comm = comm not_eq nullptr ? *comm : MPI_COMM_WORLD; + + RuntimeInst::rt = std::make_unique( + std::move(startup_config), resolved_comm, app_config, + eRuntimeInstance::DefaultInstance + ); + + auto rt_ptr = RuntimeInst::rt.get(); + if (instance == runtime::RuntimeInstType::DefaultInstance) { + // Set global variable for default instance for backward compatibility + ::vt::rt = rt_ptr; + curRT = rt_ptr; + } + RuntimeInst::rt->initialize(false, print_startup_banner); + + return runtime::makeRuntimePtr(rt_ptr); +} + template void CollectiveAnyOps::setCurrentRuntimeTLS(RuntimeUnsafePtrType in) { bool const has_rt = in != nullptr; diff --git a/src/vt/collective/collective_ops.h b/src/vt/collective/collective_ops.h index 786cee5ae7..a2fadb8f2c 100644 --- a/src/vt/collective/collective_ops.h +++ b/src/vt/collective/collective_ops.h @@ -60,10 +60,26 @@ static constexpr runtime::RuntimeInstType const collective_default_inst = template struct CollectiveAnyOps { // The general methods that interact with the managed runtime holder + + /** + * \brief Initialize VT + * + * \param[in] argc (to modify) + * \param[in] argv (to modify) + * \param[in] is_interop use interop mode (don't initialize MPI) + * \param[in] comm optional communicator + * \param[in] app_config (optional) base VT configuration to use + * \param[in] print_startup_banner (optional) whether to print startup banner + * + * \return runtime pointer + */ static RuntimePtrType initialize( int& argc, char**& argv, bool is_interop = false, MPI_Comm* comm = nullptr, - arguments::AppConfig const* appConfig = nullptr + arguments::AppConfig const* appConfig = nullptr, + bool print_startup_banner = true ); + + //// Old version of initialize with number of worker threads [[deprecated]] static RuntimePtrType initialize( int& argc, char**& argv, PhysicalResourceType const /* num_workers */, bool is_interop = false, MPI_Comm* comm = nullptr, @@ -72,6 +88,24 @@ struct CollectiveAnyOps { { return initialize(argc, argv, is_interop, comm, appConfig); } + + /** + * \brief Initialize a VT runtime with arguments preconfigured + * + * \param[in] startup_config startup config returned from \c preconfigure + * \param[in] comm the communicator for VT to use + * \param[in] app_config the app config for overriding the config + * \param[in] print_startup_banner (optional) whether to print startup banner + * + * \return the runtime pointer + */ + static RuntimePtrType initializePreconfigured( + std::unique_ptr startup_config, + MPI_Comm* comm = nullptr, + arguments::AppConfig const* app_config = nullptr, + bool print_startup_banner = true + ); + static void finalize(RuntimePtrType in_rt = nullptr); static void scheduleThenFinalize(RuntimePtrType in_rt = nullptr); static void setCurrentRuntimeTLS(RuntimeUnsafePtrType in_rt = nullptr); diff --git a/src/vt/collective/startup.cc b/src/vt/collective/startup.cc index e9a6677d99..33f382f04f 100644 --- a/src/vt/collective/startup.cc +++ b/src/vt/collective/startup.cc @@ -46,37 +46,39 @@ #include "vt/collective/collective_ops.h" #include "vt/runtime/runtime_headers.h" #include "vt/context/context.h" +#include "vt/configs/arguments/args.h" namespace vt { -std::unique_ptr -preconfigure(int& argc, char**& argv) { - return std::make_unique(argc, argv); +std::unique_ptr preconfigure( + int& argc, char**& argv +) { + auto arg_config = std::make_unique(); + auto parse_input_holder = arg_config->setupInputHolder(argc, argv); + return std::make_unique( + std::move(arg_config), std::move(parse_input_holder) + ); } RuntimePtrType initializePreconfigured( - MPI_Comm* comm, arguments::AppConfig const* appConfig, - arguments::ArgvContainer const* preconfigure_args + std::unique_ptr startup_config, + MPI_Comm* comm, + arguments::AppConfig const* app_config, + bool print_startup_banner ) { - arguments::ArgvContainer args = - preconfigure_args ? *preconfigure_args : arguments::ArgvContainer{}; - - auto argc = args.getArgc(); - auto argv_container = args.getArgvDeepCopy(); - auto argv = argv_container.get(); - bool const is_interop = comm != nullptr; - return CollectiveOps::initialize( - argc, argv, is_interop, comm, appConfig + return CollectiveOps::initializePreconfigured( + std::move(startup_config), comm, app_config, print_startup_banner ); } // vt::{initialize,finalize} for main ::vt namespace RuntimePtrType initialize( - int& argc, char**& argv, MPI_Comm* comm, arguments::AppConfig const* appConfig + int& argc, char**& argv, MPI_Comm* comm, arguments::AppConfig const* appConfig, + bool print_startup_banner ) { bool const is_interop = comm != nullptr; return CollectiveOps::initialize( - argc, argv, is_interop, comm, appConfig + argc, argv, is_interop, comm, appConfig, print_startup_banner ); } diff --git a/src/vt/collective/startup.h b/src/vt/collective/startup.h index a4f24c9f61..3dab0090c3 100644 --- a/src/vt/collective/startup.h +++ b/src/vt/collective/startup.h @@ -46,28 +46,73 @@ #include "vt/config.h" #include "vt/runtime/runtime_headers.h" -#include "vt/configs/arguments/argv_container.h" +#include "vt/collective/startup_config.h" #include +#include + namespace vt { -std::unique_ptr -preconfigure(int& argc, char**& argv); +/** + * \brief Preconfigure VT with argc/argv. This will remove all VT arguments and + * create a \c StartupConfig for VT that should be passed to \c + * initializePreconfigured. Optionally, one may specify an MPI communicator to + * use (otherwise, it defaults to \c MPI_COMM_WORLD). + * + * \note MPI must be initialized to call this function because if an error + * occurs it uses MPI rank to limit how many times the error text gets printed. + * + * \param[in] argc argc (modifies it to remove VT arguments) + * \param[in] argv argv (modifies it to remove VT arguments) + * + * \return the \c StartupConfig to pass to VT + */ +std::unique_ptr preconfigure( + int& argc, char**& argv +); + +/** + * \brief Initialize VT after it has been preconfigured + * + * \param[in] startup_config the arg config + * \param[in] comm optional communicator + * \param[in] app_config (optional) base VT configuration to use + * \param[in] print_startup_banner (optional) whether to print startup banner + * + * \return the runtime pointer + */ RuntimePtrType initializePreconfigured( + std::unique_ptr startup_config, MPI_Comm* comm = nullptr, - arguments::AppConfig const* appConfig = nullptr, - arguments::ArgvContainer const* preconfigure_args = nullptr); + arguments::AppConfig const* app_config = nullptr, + bool print_startup_banner = true +); +/** + * \brief Initialize VT + * + * \param[in] argc (to modify) + * \param[in] argv (to modify) + * \param[in] comm optional communicator + * \param[in] app_config (optional) base VT configuration to use + * \param[in] print_startup_banner (optional) whether to print startup banner + * + * \return the runtime pointer + */ RuntimePtrType initialize( int& argc, char**& argv, MPI_Comm* comm = nullptr, - arguments::AppConfig const* appConfig = nullptr + arguments::AppConfig const* appConfig = nullptr, + bool print_startup_banner = true ); + RuntimePtrType initialize(MPI_Comm* comm = nullptr); + RuntimePtrType initialize( int& argc, char**& argv, arguments::AppConfig const* appConfig ); void finalize(RuntimePtrType in_rt); + void finalize(); } /* end namespace vt */ diff --git a/src/vt/collective/startup_config.cc b/src/vt/collective/startup_config.cc new file mode 100644 index 0000000000..3c7ef9cbb2 --- /dev/null +++ b/src/vt/collective/startup_config.cc @@ -0,0 +1,58 @@ +/* +//@HEADER +// ***************************************************************************** +// +// startup_config.cc +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2024 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include "vt/collective/startup_config.h" +#include "vt/configs/arguments/args.h" + +namespace vt { + +StartupConfig::StartupConfig( + std::unique_ptr in_arg_config, + std::unique_ptr in_parse_input_holder +) : arg_config_(std::move(in_arg_config)), + parse_input_holder_(std::move(in_parse_input_holder)) +{ } + +StartupConfig::~StartupConfig() = default; + +} /* end namespace vt */ diff --git a/src/vt/configs/arguments/argv_container.h b/src/vt/collective/startup_config.h similarity index 57% rename from src/vt/configs/arguments/argv_container.h rename to src/vt/collective/startup_config.h index a3ea91140c..90f630f1c3 100644 --- a/src/vt/configs/arguments/argv_container.h +++ b/src/vt/collective/startup_config.h @@ -2,7 +2,7 @@ //@HEADER // ***************************************************************************** // -// argv_container.h +// startup_config.h // DARMA/vt => Virtual Transport // // Copyright 2019-2024 National Technology & Engineering Solutions of Sandia, LLC @@ -41,71 +41,60 @@ //@HEADER */ -#if !defined INCLUDED_VT_CONFIGS_ARGUMENTS_ARGV_CONTAINER_H -#define INCLUDED_VT_CONFIGS_ARGUMENTS_ARGV_CONTAINER_H +#if !defined INCLUDED_VT_COLLECTIVE_STARTUP_CONFIG_H +#define INCLUDED_VT_COLLECTIVE_STARTUP_CONFIG_H #include -#include -#include + +namespace vt::arguments { + +struct ParseInputHolder; +struct ArgConfig; + +} /* end namespace vt::arguments */ + +namespace vt::runtime { + +struct Runtime; + +} /* end namespace vt::runtime */ namespace vt { -namespace arguments { - -struct ArgvContainer { - ArgvContainer(int& argc, char**& argv) - { - std::vector non_vt_args; - for(int i = 0; i < argc; i++) { - // cache original argv parameter - argv_.push_back(strdup(argv[i])); - // collect non vt params - if (!((0 == strncmp(argv[i], "--vt_", 5)) || - (0 == strncmp(argv[i], "!--vt_", 6)))) { - non_vt_args.push_back(argv[i]); - } - } - - // Reconstruct argv without vt related params - int new_argc = non_vt_args.size(); - static std::unique_ptr new_argv = nullptr; - - new_argv = std::make_unique(new_argc + 1); - - int i = 0; - for (auto&& arg : non_vt_args) { - new_argv[i++] = arg; - } - new_argv[i++] = nullptr; - - argc = new_argc; - argv = new_argv.get(); - } - - ArgvContainer() = default; - ArgvContainer(const ArgvContainer&) = default; - - int getArgc() const { - return argv_.size(); - } - - std::unique_ptr getArgvDeepCopy() const { - auto output = std::make_unique(argv_.size() + 1); - - int i = 0; - for(auto&& arg : argv_) { - output[i++] = strdup(arg.c_str()); - } - output[i++] = nullptr; - - return output; - } +struct StartupConfig; + +// fwd-decl preconfigure function for friend decl +std::unique_ptr preconfigure( + int& argc, char**& argv +); + +/** + * \brief StartupConfig for preconfiguring VT before starting up the runtime. + */ +struct StartupConfig { + friend std::unique_ptr preconfigure( + int& argc, char**& argv + ); + + friend struct vt::runtime::Runtime; + + using ParseInputHolder = arguments::ParseInputHolder; + + /** + * \brief \internal Construct a \c StartupConfig + */ + StartupConfig( + std::unique_ptr in_arg_config, + std::unique_ptr in_parse_input_holder + ); + + ~StartupConfig(); private: - std::vector argv_; + std::unique_ptr arg_config_; + std::unique_ptr parse_input_holder_; }; -} // namespace arguments -} // namespace vt +} /* end namespace vt */ -#endif /*INCLUDED_VT_CONFIGS_ARGUMENTS_ARGV_CONTAINER_H*/ +#endif /*INCLUDED_VT_COLLECTIVE_STARTUP_CONFIG_H*/ diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index 39b31d4fd3..ef640e27fa 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -256,49 +256,15 @@ void parseYaml(AppConfig& appConfig, std::string const& inputFile); void convertConfigToString(CLI::App& app, AppConfig& appConfig); std::tuple parseArguments( - CLI::App& app, int& argc, char**& argv, AppConfig& appConfig + CLI::App& app, std::unique_ptr pih, AppConfig& appConfig ) { - - std::vector vt_args; - - // Load up vectors - // Has the ability to read interleaved arguments for vt, MPI, and others passed through to other libraries or the application. - std::vector* rargs = nullptr; - for (int i = 1; i < argc; i++) { - char* c = argv[i]; - if (0 == strcmp(c, "--vt_args")) { - rargs = &vt_args; - } else if (0 == strcmp(c, "--")) { - rargs = &appConfig.passthru_args; - } else if (rargs) { - rargs->push_back(c); - } else if (0 == strncmp(c, "--vt_", 5)) { - // Implicit start of VT args allows pass-thru 'for compatibility' - // although the recommended calling pattern to always provide VT args first. - rargs = &vt_args; - rargs->push_back(c); - } else { - appConfig.passthru_args.push_back(c); - } - } - // All must be accounted for app.allow_extras(false); - // Build string-vector and reverse order to parse (CLI quirk) - std::vector args_to_parse, yaml_input_arg; - for (auto it = vt_args.crbegin(); it != vt_args.crend(); ++it) { - if (util::demangle::DemanglerUtils::splitString(*it,'=')[0] == "--vt_input_config_yaml") { - yaml_input_arg.push_back(*it); - } else { - args_to_parse.push_back(*it); - } - } - // Identify input YAML file first, if present - if (!yaml_input_arg.empty()) { + if (!pih->vt_yaml_input_arg.empty()) { try { - app.parse(yaml_input_arg); + app.parse(pih->vt_yaml_input_arg); } catch (CLI::Error &ex) { // Return exit code and message, delaying logic processing of such. // The default exit code for 'help' is 0. @@ -315,7 +281,7 @@ std::tuple parseArguments( // Then parse the remaining arguments try { - app.parse(args_to_parse); + app.parse(pih->vt_args_to_parse); } catch (CLI::Error &ex) { std::stringstream message_stream; int result = app.exit(ex, message_stream, message_stream); @@ -327,42 +293,8 @@ std::tuple parseArguments( if (appConfig.vt_output_config) { convertConfigToString(app, appConfig); } - - // Get the clean prog name; don't allow path bleed in usages. - // std::filesystem is C++17. - std::string clean_prog_name = argv[0]; - size_t l = clean_prog_name.find_last_of("/\\"); - if (l not_eq std::string::npos and l + 1 < clean_prog_name.size()) { - clean_prog_name = clean_prog_name.substr(l + 1, std::string::npos); - } - - appConfig.prog_name = clean_prog_name; - appConfig.argv_prog_name = argv[0]; - postParseTransform(appConfig); - // Rebuild passthru into ref-returned argc/argv - - // It should be possible to modify the original argv as the outgoing - // number of arguments is always less. As currently allocated here, - // ownership of the new object is ill-defined. - int new_argc = appConfig.passthru_args.size() + 1; // does not include argv[0] - - static std::unique_ptr new_argv = nullptr; - - new_argv = std::make_unique(new_argc + 1); - - int i = 0; - new_argv[i++] = appConfig.argv_prog_name; - for (auto&& arg : appConfig.passthru_args) { - new_argv[i++] = arg; - } - new_argv[i++] = nullptr; - - // Set them back with all vt (and MPI) arguments elided - argc = new_argc; - argv = new_argv.get(); - return std::make_tuple(-1, std::string{}); } @@ -1430,17 +1362,101 @@ class VtFormatter : public CLI::Formatter { } }; +std::unique_ptr ArgConfig::setupInputHolder( + int& argc, char**& argv +) { + auto pih = std::make_unique(); + + if (argc == 0) { + return nullptr; + } + + std::vector vt_args; + + // Load up vectors + // Has the ability to read interleaved arguments for vt, MPI, and others passed through to other libraries or the application. + std::vector* rargs = nullptr; + for (int i = 1; i < argc; i++) { + char* c = argv[i]; + if (0 == strcmp(c, "--vt_args")) { + rargs = &vt_args; + } else if (0 == strcmp(c, "--")) { + rargs = &pih->passthru_args; + } else if (rargs) { + rargs->push_back(c); + } else if (0 == strncmp(c, "--vt_", 5)) { + // Implicit start of VT args allows pass-thru 'for compatibility' + // although the recommended calling pattern to always provide VT args first. + rargs = &vt_args; + rargs->push_back(c); + } else { + pih->passthru_args.push_back(c); + } + } + + // Build string-vector and reverse order to parse (CLI quirk) + for (auto it = vt_args.crbegin(); it != vt_args.crend(); ++it) { + if (util::demangle::DemanglerUtils::splitString(*it,'=')[0] == "--vt_input_config_yaml") { + pih->vt_yaml_input_arg.push_back(*it); + } else { + pih->vt_args_to_parse.push_back(*it); + } + } + + // Get the clean prog name; don't allow path bleed in usages. + // std::filesystem is C++17. + std::string clean_prog_name = argv[0]; + size_t l = clean_prog_name.find_last_of("/\\"); + if (l not_eq std::string::npos and l + 1 < clean_prog_name.size()) { + clean_prog_name = clean_prog_name.substr(l + 1, std::string::npos); + } + + pih->clean_prog_name = clean_prog_name; + pih->argv_prog_name = argv[0]; + + // Rebuild passthru into ref-returned argc/argv + + // It should be possible to modify the original argv as the outgoing + // number of arguments is always less. As currently allocated here, + // ownership of the new object is ill-defined. + int new_argc = pih->passthru_args.size() + 1; // does not include argv[0] + + static std::unique_ptr new_argv = nullptr; + + new_argv = std::make_unique(new_argc + 1); + + int i = 0; + new_argv[i++] = pih->argv_prog_name; + for (auto&& arg : pih->passthru_args) { + new_argv[i++] = arg; + } + new_argv[i++] = nullptr; + + // Set them back with all vt (and MPI) arguments elided + argc = new_argc; + argv = new_argv.get(); + + return pih; +} + std::tuple ArgConfig::parse( int& argc, char**& argv, AppConfig const* appConfig +) { + return parse(setupInputHolder(argc, argv), appConfig); +} + +std::tuple ArgConfig::parse( + std::unique_ptr pih, + AppConfig const* appConfig ) { // If user didn't define appConfig, parse into this->config_. if (not appConfig) { - return parseToConfig(argc, argv, config_); + return parseToConfig(std::move(pih), config_); } // If user defines appConfig, parse into temporary config for later comparison. AppConfig config{*appConfig}; - auto const parse_result = parseToConfig(argc, argv, config); + auto const parse_result = parseToConfig(std::move(pih), config); config_ = config; @@ -1448,9 +1464,19 @@ std::tuple ArgConfig::parse( } std::tuple ArgConfig::parseToConfig( - int& argc, char**& argv, AppConfig& appConfig + std::unique_ptr pih, AppConfig& appConfig ) { - if (parsed_ || argc == 0 || argv == nullptr) { + if (pih == nullptr) { + return std::make_tuple(-1, std::string{}); + } + + bool const no_input_args = + pih->vt_args_to_parse.size() == 0 and pih->vt_yaml_input_arg.size() == 0; + + appConfig.prog_name = pih->clean_prog_name; + appConfig.argv_prog_name = pih->argv_prog_name; + + if (parsed_ || no_input_args) { // Odd case.. pretend nothing bad happened. return std::make_tuple(-1, std::string{}); } @@ -1488,7 +1514,9 @@ std::tuple ArgConfig::parseToConfig( addTVArgs(app, appConfig); addThreadingArgs(app, appConfig); - std::tuple result = parseArguments(app, argc, argv, appConfig); + std::tuple result = parseArguments( + app, std::move(pih), appConfig + ); if (std::get<0>(result) not_eq -1) { // non-success return result; diff --git a/src/vt/configs/arguments/args.h b/src/vt/configs/arguments/args.h index c9ab8d7e10..002fbbcd22 100644 --- a/src/vt/configs/arguments/args.h +++ b/src/vt/configs/arguments/args.h @@ -61,6 +61,20 @@ class App; namespace arguments { +/** + * \struct ParseInputHolder + * + * \brief Internal structure to hold all the inputs for setting up a correct + * \c AppConfig from command line arguments. + */ +struct ParseInputHolder { + std::vector vt_args_to_parse; + std::vector vt_yaml_input_arg; + std::string clean_prog_name; + char* argv_prog_name = nullptr; + std::vector passthru_args; +}; + /** * \struct ArgConfig * @@ -77,6 +91,21 @@ struct ArgConfig : runtime::component::Component { int& argc, char**& argv, AppConfig const* appConfig ); + std::tuple parse( + std::unique_ptr parse_input_holder, + AppConfig const* appConfig + ); + + /** + * \brief Setup the input holder and modify argc/argv + * + * \param[in] argc argc + * \param[in] argv argv + * + * \return the parse input holder + */ + std::unique_ptr setupInputHolder(int& argc, char**& argv); + static std::unique_ptr construct(std::unique_ptr arg); std::string name() override { return "ArgConfig"; } @@ -91,7 +120,7 @@ struct ArgConfig : runtime::component::Component { private: std::tuple parseToConfig( - int& argc, char**& argv, AppConfig& appConfig + std::unique_ptr pih, AppConfig& appConfig ); bool parsed_ = false; diff --git a/src/vt/runtime/runtime.cc b/src/vt/runtime/runtime.cc index cbab82a9e3..ab81af0327 100644 --- a/src/vt/runtime/runtime.cc +++ b/src/vt/runtime/runtime.cc @@ -96,6 +96,25 @@ namespace vt { namespace runtime { /*static*/ bool volatile Runtime::sig_user_1_ = false; +Runtime::Runtime( + std::unique_ptr startup_config, + MPI_Comm in_comm, + arguments::AppConfig const* appConfig, + RuntimeInstType const in_instance +) : instance_(in_instance), runtime_active_(false), is_interop_(true), + initial_communicator_(in_comm), + arg_config_(std::move(startup_config->arg_config_)), + app_config_(&arg_config_->config_) +{ + startupMPIConfigArgs( + std::move(startup_config->parse_input_holder_), + initial_communicator_, arg_config_.get(), + appConfig + ); + setUpSignals(); + determinePhysicalNodeIDs(); +} + Runtime::Runtime( int& argc, char**& argv, bool const interop_mode, MPI_Comm in_comm, RuntimeInstType const in_instance, arguments::AppConfig const* appConfig @@ -132,10 +151,29 @@ Runtime::Runtime( /// /// ========================================================================= + startupMPIConfigArgs( + argc, argv, is_interop_, initial_communicator_, arg_config_.get(), + appConfig + ); + setUpSignals(); + determinePhysicalNodeIDs(); +} + +void Runtime::setUpSignals() { + sig_user_1_ = false; + setupSignalHandler(); + setupSignalHandlerINT(); + setupTerminateHandler(); +} + +/*static*/ void Runtime::startupMPIConfigArgs( + int& argc, char**& argv, bool is_interop, MPI_Comm in_comm, + arguments::ArgConfig* arg_config, arguments::AppConfig const* appConfig +) { int prev_initialized; MPI_Initialized(&prev_initialized); - if (not is_interop_) { + if (not is_interop) { vtAbortIf( prev_initialized, "MPI is already initialzed. Run VT under interop-mode?" ); @@ -148,20 +186,35 @@ Runtime::Runtime( ); } + startupMPIConfigArgs( + arg_config->setupInputHolder(argc, argv), + in_comm, + arg_config, + appConfig + ); +} + +/*static*/ void Runtime::startupMPIConfigArgs( + std::unique_ptr pih, + MPI_Comm in_comm, arguments::ArgConfig* arg_config, + arguments::AppConfig const* appConfig +) { // n.b. ref-update of args with pass-through arguments std::tuple result = - arg_config_->parse(argc, argv, appConfig); + arg_config->parse(std::move(pih), appConfig); int exit_code = std::get<0>(result); - if (getAppConfig()->vt_help_lb_args) { + if (arg_config->config_.vt_help_lb_args) { // Help requested or invalid argument(s). int rank = 0; - MPI_Comm_rank(initial_communicator_, &rank); + MPI_Comm_rank(in_comm, &rank); if (rank == 0) { // Help requested vt::debug::preConfigRef()->colorize_output = true; - vrt::collection::balance::LBManager::printLBArgsHelp(getAppConfig()->vt_lb_name); + vrt::collection::balance::LBManager::printLBArgsHelp( + arg_config->config_.vt_lb_name + ); } if (exit_code == -1) { exit_code = 0; @@ -170,7 +223,7 @@ Runtime::Runtime( if (exit_code not_eq -1) { // Help requested or invalid argument(s). - MPI_Comm comm = initial_communicator_; + MPI_Comm comm = in_comm; int rank = 0; MPI_Comm_rank(comm, &rank); @@ -199,51 +252,44 @@ Runtime::Runtime( std::_Exit(exit_code); // no return } - - sig_user_1_ = false; - setupSignalHandler(); - setupSignalHandlerINT(); - setupTerminateHandler(); - - if (arg_config_->config_.vt_lb_data) { - determinePhysicalNodeIDs(); - } } void Runtime::determinePhysicalNodeIDs() { - MPI_Comm i_comm = initial_communicator_; + if (arg_config_->config_.vt_lb_data) { + MPI_Comm i_comm = initial_communicator_; - MPI_Comm shm_comm; - MPI_Comm_split_type(i_comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shm_comm); - int shm_rank = -1; - int node_size = -1; - MPI_Comm_rank(shm_comm, &shm_rank); - MPI_Comm_size(shm_comm, &node_size); + MPI_Comm shm_comm; + MPI_Comm_split_type(i_comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shm_comm); + int shm_rank = -1; + int node_size = -1; + MPI_Comm_rank(shm_comm, &shm_rank); + MPI_Comm_size(shm_comm, &node_size); - int num_nodes = -1; - int is_rank_0 = (shm_rank == 0) ? 1 : 0; - MPI_Allreduce(&is_rank_0, &num_nodes, 1, MPI_INT, MPI_SUM, i_comm); + int num_nodes = -1; + int is_rank_0 = (shm_rank == 0) ? 1 : 0; + MPI_Allreduce(&is_rank_0, &num_nodes, 1, MPI_INT, MPI_SUM, i_comm); - int starting_rank = -1; - MPI_Comm_rank(i_comm, &starting_rank); + int starting_rank = -1; + MPI_Comm_rank(i_comm, &starting_rank); - MPI_Comm node_number_comm; - MPI_Comm_split(i_comm, shm_rank, starting_rank, &node_number_comm); + MPI_Comm node_number_comm; + MPI_Comm_split(i_comm, shm_rank, starting_rank, &node_number_comm); - int node_id = -1; - if (shm_rank == 0) { - MPI_Comm_rank(node_number_comm, &node_id); - } - MPI_Bcast(&node_id, 1, MPI_INT, 0, shm_comm); + int node_id = -1; + if (shm_rank == 0) { + MPI_Comm_rank(node_number_comm, &node_id); + } + MPI_Bcast(&node_id, 1, MPI_INT, 0, shm_comm); - MPI_Comm_free(&shm_comm); - MPI_Comm_free(&node_number_comm); + MPI_Comm_free(&shm_comm); + MPI_Comm_free(&node_number_comm); - has_physical_node_info = true; - physical_node_id = node_id; - physical_num_nodes = num_nodes; - physical_node_size = node_size; - physical_node_rank = shm_rank; + has_physical_node_info = true; + physical_node_id = node_id; + physical_num_nodes = num_nodes; + physical_node_size = node_size; + physical_node_rank = shm_rank; + } } bool Runtime::hasSchedRun() const { @@ -448,7 +494,7 @@ bool Runtime::needLBDataRestartReader() { return needOfflineLB; } -bool Runtime::initialize(bool const force_now) { +bool Runtime::initialize(bool const force_now, bool print_startup_banner) { if (force_now) { initializeComponents(); initializeOptionalComponents(); @@ -458,7 +504,9 @@ bool Runtime::initialize(bool const force_now) { MPI_Barrier(comm); if (theContext->getNode() == 0) { - printStartupBanner(); + if (print_startup_banner) { + printStartupBanner(); + } // Enqueue a check for later in case arguments are modified before work // actually executes theSched->enqueue([this]{ diff --git a/src/vt/runtime/runtime.h b/src/vt/runtime/runtime.h index 2cf721d33d..b4dc772f8d 100644 --- a/src/vt/runtime/runtime.h +++ b/src/vt/runtime/runtime.h @@ -49,6 +49,7 @@ #include "vt/runtime/runtime_component_fwd.h" #include "vt/runtime/component/component_pack.h" #include "vt/timing/timing_type.h" +#include "vt/collective/startup_config.h" // Optional components #if vt_check_enabled(trace_enabled) @@ -107,12 +108,66 @@ struct Runtime { arguments::AppConfig const* appConfig = nullptr ); + /** + * \internal \brief Initialize a VT runtime with app config + * + * Under interop mode, MPI is not initialized or finalized by the runtime. + * This can be used to embed VT into a larger context. + * + * When not running in interop mode, MPI is initialized in the constructor + * and finalized in the destructor. + * + * \param[in] startup_config startup configuration + * \param[in] in_comm the MPI communicator (if in interoperability mode) + * \param[in] app_config optional app configuration + * \param[in] in_instance the runtime instance to set + */ + Runtime( + std::unique_ptr startup_config, + MPI_Comm in_comm = MPI_COMM_WORLD, + arguments::AppConfig const* appConfig = nullptr, + RuntimeInstType const in_instance = RuntimeInstType::DefaultInstance + ); + Runtime(Runtime const&) = delete; Runtime(Runtime&&) = delete; Runtime& operator=(Runtime const&) = delete; virtual ~Runtime(); + void setUpSignals(); + + /** + * \brief Startup MPI if necessary and configure VT arguments based on \c argc + * and \c argv + * + * \param[in] argc argc (to modify) + * \param[in] argv argv (to modify) + * \param[in] is_interop whether we are running in interop mode + * \param[in] in_comm the comm + * \param[in] arg_config the arg config to fill + * \param[in] appConfig possible app config overrides + */ + static void startupMPIConfigArgs( + int& argc, char**& argv, bool is_interop, MPI_Comm in_comm, + arguments::ArgConfig* arg_config, arguments::AppConfig const* appConfig + ); + + /** + * \brief Startup MPI if necessary and configure VT arguments based on parse + * input holder + * + * \param[in] pih parse input holder + * \param[in] in_comm the comm + * \param[in] arg_config the arg config to fill + * \param[in] appConfig possible app config overrides + */ + static void startupMPIConfigArgs( + std::unique_ptr pih, + MPI_Comm in_comm, arguments::ArgConfig* arg_config, + arguments::AppConfig const* appConfig + ); + /** * \brief Check if runtime is live * @@ -180,10 +235,11 @@ struct Runtime { * \internal \brief Initialize the runtime * * \param[in] force_now whether to force initialization regardless of state + * \param[in] print_startup_banner whether to print startup banner * * \return whether it initialized or not */ - bool initialize(bool const force_now = false); + bool initialize(bool const force_now = false, bool print_startup_banner = true); /** * \internal \brief Finalize the runtime @@ -239,6 +295,11 @@ struct Runtime { */ void systemSync(); + /** + * \brief Print a very informative startup banner + */ + void printStartupBanner(); + public: /** * \internal \brief Check for input argument errors @@ -331,11 +392,6 @@ struct Runtime { */ void terminationHandler(); - /** - * \internal \brief Print a very informative startup banner - */ - void printStartupBanner(); - /** * \internal \brief Print the shutdown banner * diff --git a/tests/unit/runtime/test_initialization.cc b/tests/unit/runtime/test_initialization.cc index e3d35b7e1f..f54b5742f5 100644 --- a/tests/unit/runtime/test_initialization.cc +++ b/tests/unit/runtime/test_initialization.cc @@ -853,4 +853,41 @@ TEST_F(TestInitialization, test_initialize_with_yaml_toml_and_args) { EXPECT_EQ(theConfig()->vt_debug_level, "verbose"); // args overwrite everything } +struct TestInitializationPreConfig : TestHarness { }; + +TEST_F(TestInitializationPreConfig, test_vt_preconfigure_args_1) { + static char prog_name[]{"vt_program"}; + static char random_argument[]{"--random_argument=100"}; +#if vt_check_enabled(lblite) + static char vt_lb[]{"--vt_lb"}; + static char vt_lb_name[]{"--vt_lb_name=TemperedLB"}; +#endif + + std::vector custom_args = { + prog_name, random_argument, +#if vt_check_enabled(lblite) + vt_lb, vt_lb_name +#endif + }; + + int argc = static_cast(custom_args.size()); + char** argv = custom_args.data(); + + auto startup_config = vt::preconfigure(argc, argv); + + MPI_Init(&argc, &argv); + + EXPECT_EQ(argc, 2); + + vt::initializePreconfigured(std::move(startup_config)); + +#if vt_check_enabled(lblite) + EXPECT_TRUE(theConfig()->vt_lb); + EXPECT_EQ(theConfig()->vt_lb_name, "TemperedLB"); +#endif + + vt::finalize(); + MPI_Finalize(); +} + }}} // end namespace vt::tests::unit