diff --git a/doc/src/helper_functions.rst b/doc/src/helper_functions.rst index 2b85eeb13..6723fd7b0 100644 --- a/doc/src/helper_functions.rst +++ b/doc/src/helper_functions.rst @@ -17,17 +17,6 @@ are documented in `the admm wrapper documentation = mk_m1, "Our sample size should be increasing" if (k%self.kf_xhat==0): #We use only new scenarios to compute xhat - xhat_scenario_names = refmodel.scenario_names_creator(int(mult*nk), + xhat_scenario_names = scenario_names_creator(int(mult*nk), start=self.ScenCount) self.ScenCount+= mk else: #We reuse the previous scenarios - xhat_scenario_names+= refmodel.scenario_names_creator(mult*(nk-nk_m1), + xhat_scenario_names+= scenario_names_creator(mult*(nk-nk_m1), start=self.ScenCount) self.ScenCount+= mk-mk_m1 @@ -474,20 +470,18 @@ def run(self,maxit=200): gap_branching_factors = ciutils.scalable_branching_factors(lower_bound_k, lcfg['branching_factors']) nk = np.prod(gap_branching_factors) - estimator_scenario_names = refmodel.scenario_names_creator(nk) + estimator_scenario_names = scenario_names_creator(nk) sample_options = {'branching_factors':gap_branching_factors, 'seed':self.SeedCount} else: nk = self.ArRP *int(np.ceil(lower_bound_k/self.ArRP)) assert nk>= nk_m1, "Our sample size should be increasing" if (k%self.kf_GS==0): #We use only new scenarios to compute gap estimators - estimator_scenario_names = refmodel.scenario_names_creator(nk, - start=self.ScenCount) + estimator_scenario_names = scenario_names_creator(nk, start=self.ScenCount) self.ScenCount+=nk else: #We reuse the previous scenarios - estimator_scenario_names+= refmodel.scenario_names_creator((nk-nk_m1), - start=self.ScenCount) + estimator_scenario_names+= scenario_names_creator((nk-nk_m1), start=self.ScenCount) self.ScenCount+= (nk-nk_m1) sample_options = None diff --git a/mpisppy/generic_cylinders.py b/mpisppy/generic_cylinders.py index 83e276a26..4a2bf5969 100644 --- a/mpisppy/generic_cylinders.py +++ b/mpisppy/generic_cylinders.py @@ -11,20 +11,17 @@ import sys import os import copy +import glob import numpy as np import pyomo.environ as pyo import pyomo.common.config as pyofig from mpisppy.spin_the_wheel import WheelSpinner -import mpisppy.utils.cfg_vanilla as vanilla -import mpisppy.utils.config as config -import mpisppy.utils.sputils as sputils +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator, solver_spec, sputils from mpisppy.extensions.extension import MultiExtension, Extension -import mpisppy.utils.solver_spec as solver_spec - from mpisppy import global_toc from mpisppy import MPI @@ -102,7 +99,7 @@ def _parse_args(m): cfg.checker() # looks for inconsistencies return cfg -def _name_lists(module, cfg, bundle_wrapper=None): +def _name_lists(cfg, bundle_wrapper=None): # Note: high level code like this assumes there are branching factors for # multi-stage problems. For other trees, you will need lower-level code @@ -114,7 +111,10 @@ def _name_lists(module, cfg, bundle_wrapper=None): "For now, stage2EFsolvern is required for multistage xhat" else: all_nodenames = None - num_scens = cfg.get("num_scens") # maybe None is OK + num_scens = cfg.get("num_scens") + if num_scens is None: + mps_files = glob.glob(os.path.join(cfg.get("mps_files_directory"), "*.mps")) + num_scens = len(mps_files) # proper bundles should be almost magic if cfg.unpickle_bundles_dir or cfg.scenarios_per_bundle is not None: @@ -122,7 +122,7 @@ def _name_lists(module, cfg, bundle_wrapper=None): all_scenario_names = bundle_wrapper.bundle_names_creator(num_buns, cfg=cfg) all_nodenames = None # This is seldom used; also, proper bundles result in two stages else: - all_scenario_names = module.scenario_names_creator(num_scens) + all_scenario_names = scenario_names_creator(num_scens, prefix='scen') return all_scenario_names, all_nodenames @@ -163,7 +163,7 @@ def do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_d else: ph_converger = None - all_scenario_names, all_nodenames = _name_lists(module, cfg, bundle_wrapper=bundle_wrapper) + all_scenario_names, all_nodenames = _name_lists(cfg, bundle_wrapper=bundle_wrapper) # Things needed for vanilla cylinders beans = (cfg, scenario_creator, scenario_denouement, all_scenario_names) @@ -482,10 +482,8 @@ def _write_scenarios(module, local_slice = slices[my_rank] my_start = local_slice[0] # zero based - inum = sputils.extract_num(module.scenario_names_creator(1)[0]) - - local_scenario_names = module.scenario_names_creator(len(local_slice), - start=inum + my_start) + inum = sputils.extract_num(scenario_names_creator(1)[0]) + local_scenario_names = scenario_names_creator(len(local_slice), start=inum + my_start) if my_rank == 0: if os.path.exists(cfg.pickle_scenarios_dir): shutil.rmtree(cfg.pickle_scenarios_dir) @@ -533,7 +531,7 @@ def _write_bundles(module, local_slice = slices[my_rank] # We need to know if scenarios (not bundles) are one-based. - inum = sputils.extract_num(module.scenario_names_creator(1)[0]) + inum = sputils.extract_num(scenario_names_creator(1)[0]) local_bundle_names = [f"Bundle_{bn*bsize+inum}_{(bn+1)*bsize-1+inum}" for bn in local_slice] @@ -547,14 +545,14 @@ def _write_bundles(module, fname = os.path.join(cfg.pickle_bundles_dir, bname+".pkl") pickle_bundle.dill_pickle(bundle, fname) global_toc(f"Bundles written to {cfg.pickle_bundles_dir}") - + #========== def _do_EF(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_denouement, bundle_wrapper=None): - all_scenario_names, _ = _name_lists(module, cfg, bundle_wrapper=bundle_wrapper) + all_scenario_names, _ = _name_lists(cfg, bundle_wrapper=bundle_wrapper) ef = sputils.create_EF( all_scenario_names, - module.scenario_creator, + scenario_creator, scenario_creator_kwargs=module.kw_creator(cfg), ) @@ -649,7 +647,7 @@ def _write_scenario_lp_mps_files_only(module, from mpisppy.spbase import SPBase from mpisppy.extensions.scenario_lp_mps_files import Scenario_lp_mps_files - all_scenario_names, all_nodenames = _name_lists(module, cfg, bundle_wrapper=bundle_wrapper) + all_scenario_names, all_nodenames = _name_lists(cfg, bundle_wrapper=bundle_wrapper) # SPBase builds the scenarios and scenario-tree bookkeeping; no solves happen here. sp_options = { diff --git a/mpisppy/opt/ph.py b/mpisppy/opt/ph.py index 982387fc3..983985fe0 100644 --- a/mpisppy/opt/ph.py +++ b/mpisppy/opt/ph.py @@ -11,6 +11,7 @@ import mpisppy.phbase import shutil import mpisppy.MPI as mpi +from mpisppy.utils import scenario_names_creator # decorator snarfed from stack overflow - allows per-rank profile output file generation. def profile(filename=None, comm=mpi.COMM_WORLD): @@ -101,7 +102,7 @@ def ph_main(self, finalize=True, attach_prox=True): PHopt["iterk_solver_options"] = {"mipgap": 0.001} ScenCount = 50 - all_scenario_names = ['scen' + str(i) for i in range(ScenCount)] + all_scenario_names = scenario_names_creator(ScenCount, prefix="scen") # end hardwire scenario_creator = refmodel.scenario_creator diff --git a/mpisppy/tests/examples/aircond.py b/mpisppy/tests/examples/aircond.py index 3ca2474a1..ae9720060 100644 --- a/mpisppy/tests/examples/aircond.py +++ b/mpisppy/tests/examples/aircond.py @@ -367,16 +367,7 @@ def sample_tree_scen_creator(sname, stage, sample_branching_factors, seed, starting_stage=stage) return model - -#========= -def scenario_names_creator(num_scens,start=None): - # (only for Amalgamator): return the full list of num_scens scenario names - # if start!=None, the list starts with the 'start' labeled scenario - if (start is None) : - start=0 - return [f"scen{i}" for i in range(start,start+num_scens)] - #========= def inparser_adder(cfg): diff --git a/mpisppy/tests/examples/aircondB.py b/mpisppy/tests/examples/aircondB.py index d48480008..33e6d6bc9 100644 --- a/mpisppy/tests/examples/aircondB.py +++ b/mpisppy/tests/examples/aircondB.py @@ -183,15 +183,6 @@ def sample_tree_scen_creator(sname, stage, sample_branching_factors, seed, given_scenario=given_scenario, **scenario_creator_kwargs) -#========= -def scenario_names_creator(num_scens,start=None): - # (only for Amalgamator): return the full list of num_scens scenario names - # if start!=None, the list starts with the 'start' labeled scenario - if (start is None) : - start=0 - return [f"scen{i}" for i in range(start,start+num_scens)] - - #========= def inparser_adder(cfg): base_aircond.inparser_adder(cfg) diff --git a/mpisppy/tests/examples/apl1p.py b/mpisppy/tests/examples/apl1p.py index f853870ef..5b78fc22f 100644 --- a/mpisppy/tests/examples/apl1p.py +++ b/mpisppy/tests/examples/apl1p.py @@ -179,15 +179,6 @@ def scenario_creator(sname, num_scens=None): return(model) -#========= -def scenario_names_creator(num_scens,start=None): - # (only for Amalgamator): return the full list of num_scens scenario names - # if start!=None, the list starts with the 'start' labeled scenario - if (start is None) : - start=0 - return [f"scen{i}" for i in range(start,start+num_scens)] - - #========= def inparser_adder(inparser): # (only for Amalgamator): add command options unique to apl1p diff --git a/mpisppy/tests/examples/distr/distr.py b/mpisppy/tests/examples/distr/distr.py index c6f81a7fd..4a211d7ae 100644 --- a/mpisppy/tests/examples/distr/distr.py +++ b/mpisppy/tests/examples/distr/distr.py @@ -378,19 +378,6 @@ def consensus_vars_creator(num_scens): consensus_vars[region_target].append(vstr) return consensus_vars - -def scenario_names_creator(num_scens): - """Creates the name of every scenario. - - Args: - num_scens (int): number of scenarios - - Returns: - list (str): the list of names - """ - return [f"Region{i+1}" for i in range(num_scens)] - - def kw_creator(cfg): """ Args: @@ -399,8 +386,7 @@ def kw_creator(cfg): Returns: dict (str): the kwargs that are used in distr.scenario_creator, here {"num_scens": num_scens} """ - kwargs = {"num_scens" : cfg.get('num_scens', None), - } + kwargs = {"num_scens" : cfg.get('num_scens', None)} if kwargs["num_scens"] not in [2, 3, 4]: RuntimeError (f"unexpected number of regions {cfg.num_scens}, whould be in [2, 3, 4]") return kwargs diff --git a/mpisppy/tests/examples/farmer.py b/mpisppy/tests/examples/farmer.py index 48a5938d4..f63efed44 100644 --- a/mpisppy/tests/examples/farmer.py +++ b/mpisppy/tests/examples/farmer.py @@ -240,16 +240,6 @@ def total_cost_rule(model): # begin functions not needed by farmer_cylinders # (but needed by special codes such as confidence intervals) -#========= -def scenario_names_creator(num_scens,start=None): - # (only for Amalgamator): return the full list of num_scens scenario names - # if start!=None, the list starts with the 'start' labeled scenario - if (start is None) : - start=0 - return [f"scen{i}" for i in range(start,start+num_scens)] - - - #========= def inparser_adder(cfg): # add options unique to farmer diff --git a/mpisppy/tests/examples/gbd/gbd.py b/mpisppy/tests/examples/gbd/gbd.py index 9502ae5e8..1caab20db 100644 --- a/mpisppy/tests/examples/gbd/gbd.py +++ b/mpisppy/tests/examples/gbd/gbd.py @@ -10,12 +10,11 @@ #From original Ferguson and Dantzig 1956 #Extended scenarios as done in Seq Sampling by B&M +import json import pyomo.environ as pyo import numpy as np import mpisppy.scenario_tree as scenario_tree -import mpisppy.utils.sputils as sputils -import mpisppy.utils.amalgamator as amalgamator -import json +from mpisppy.utils import amalgamator, sputils, scenario_names_creator # Use this random stream: gbdstream= np.random.RandomState() # pylint: disable=no-member @@ -199,15 +198,6 @@ def scenario_creator(sname, num_scens=None): return(model) -#========= -def scenario_names_creator(num_scens,start=None): - # (only for Amalgamator): return the full list of num_scens scenario names - # if start!=None, the list starts with the 'start' labeled scenario - if (start is None) : - start=0 - return [f"scen{i}" for i in range(start,start+num_scens)] - - #========= def inparser_adder(inparser): # (only for Amalgamator): add command options unique to gbd @@ -274,7 +264,7 @@ def xhat_generator_gbd(scenario_names, solver_name="gurobi", solver_options=None solver_name = 'gurobi_direct' solver_options=None - scenario_names = scenario_names_creator(num_scens,start=6000) + scenario_names = scenario_names_creator(num_scens, prefix="scen", start=6000) ama_options = { "EF-2stage": True, "EF_solver_name": solver_name, diff --git a/mpisppy/tests/examples/w_test_data/farmer_get.py b/mpisppy/tests/examples/w_test_data/farmer_get.py index 4099e7eda..e12634e8d 100644 --- a/mpisppy/tests/examples/w_test_data/farmer_get.py +++ b/mpisppy/tests/examples/w_test_data/farmer_get.py @@ -12,10 +12,8 @@ # Make it all go from mpisppy.spin_the_wheel import WheelSpinner -import mpisppy.utils.sputils as sputils +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator, sputils -from mpisppy.utils import config -import mpisppy.utils.cfg_vanilla as vanilla from mpisppy.extensions.norm_rho_updater import NormRhoUpdater from mpisppy.convergers.norm_rho_converger import NormRhoConverger @@ -83,7 +81,7 @@ def main(): scenario_creator = farmer.scenario_creator scenario_denouement = farmer.scenario_denouement - all_scenario_names = ['scen{}'.format(sn) for sn in range(num_scen)] + all_scenario_names = scenario_names_creator(num_scen, prefix='scen') scenario_creator_kwargs = { 'use_integer': False, "crops_multiplier": crops_multiplier, diff --git a/mpisppy/tests/examples/w_test_data/farmer_set.py b/mpisppy/tests/examples/w_test_data/farmer_set.py index f28373dac..60b6d8895 100644 --- a/mpisppy/tests/examples/w_test_data/farmer_set.py +++ b/mpisppy/tests/examples/w_test_data/farmer_set.py @@ -12,10 +12,7 @@ # Make it all go from mpisppy.spin_the_wheel import WheelSpinner -import mpisppy.utils.sputils as sputils - -from mpisppy.utils import config -import mpisppy.utils.cfg_vanilla as vanilla +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator, sputils from mpisppy.extensions.norm_rho_updater import NormRhoUpdater from mpisppy.convergers.norm_rho_converger import NormRhoConverger @@ -85,7 +82,7 @@ def main(): scenario_creator = farmer.scenario_creator scenario_denouement = farmer.scenario_denouement - all_scenario_names = ['scen{}'.format(sn) for sn in range(num_scen)] + all_scenario_names = scenario_names_creator(num_scen, prefix='scen') scenario_creator_kwargs = { 'use_integer': False, "crops_multiplier": crops_multiplier, diff --git a/mpisppy/tests/test_admmWrapper.py b/mpisppy/tests/test_admmWrapper.py index 3682cc277..07105d946 100644 --- a/mpisppy/tests/test_admmWrapper.py +++ b/mpisppy/tests/test_admmWrapper.py @@ -8,9 +8,8 @@ ############################################################################### # TBD: make these tests less fragile import unittest -import mpisppy.utils.admmWrapper as admmWrapper import mpisppy.tests.examples.distr.distr as distr -from mpisppy.utils import config +from mpisppy.utils import admmWrapper, config, scenario_names_creator from mpisppy.tests.utils import get_solver from mpisppy import MPI import subprocess @@ -36,7 +35,7 @@ def _cfg_creator(self, num_scens): def _make_admm(self, num_scens,verbose=False): cfg = self._cfg_creator(num_scens) options = {} - all_scenario_names = distr.scenario_names_creator(num_scens=cfg.num_scens) + all_scenario_names = scenario_names_creator(cfg.num_scens, prefix="Region", start=1) scenario_creator = distr.scenario_creator scenario_creator_kwargs = distr.kw_creator(cfg) consensus_vars = distr.consensus_vars_creator(cfg.num_scens) @@ -56,7 +55,7 @@ def test_constructor(self): for i in range(3,5): self._make_admm(i) - def test_variable_probability(self): + def test_variable_probability(self): admm = self._make_admm(3) q = dict() for sname, s in admm.local_scenarios.items(): diff --git a/mpisppy/tests/test_agnostic.py b/mpisppy/tests/test_agnostic.py index da64ef2e7..9fb3e122d 100644 --- a/mpisppy/tests/test_agnostic.py +++ b/mpisppy/tests/test_agnostic.py @@ -13,10 +13,9 @@ import math import mpisppy.opt.ph from mpisppy.tests.utils import get_solver -import mpisppy.utils.config as config -import mpisppy.agnostic.agnostic as agnostic -import mpisppy.agnostic.agnostic_cylinders as agnostic_cylinders -import mpisppy.utils.sputils as sputils +from mpisppy.utils import config, sputils, scenario_names_creator +from mpisppy.agnostic import agnostic, agnostic_cylinders + sys.path.insert(0, "../../examples/farmer/agnostic") import farmer_pyomo_agnostic @@ -34,7 +33,7 @@ import farmer_gurobipy_agnostic have_gurobipy = True except ModuleNotFoundError: - have_gurobipy = False + have_gurobipy = False __version__ = 0.2 @@ -101,7 +100,7 @@ def test_agnostic_pyomo_PH_constructor(self): phoptions = _get_ph_base_options() ph = mpisppy.opt.ph.PH( phoptions, - farmer_pyomo_agnostic.scenario_names_creator(num_scens=3), + scenario_names_creator(3), Ag.scenario_creator, farmer_pyomo_agnostic.scenario_denouement, scenario_creator_kwargs=None, # agnostic.py takes care of this @@ -116,7 +115,7 @@ def test_agnostic_pyomo_PH(self): s1 = Ag.scenario_creator("scen1") # average case phoptions = _get_ph_base_options() phoptions["Ag"] = Ag # this is critical - scennames = farmer_pyomo_agnostic.scenario_names_creator(num_scens=3) + scennames = scenario_names_creator(3) ph = mpisppy.opt.ph.PH( phoptions, scennames, @@ -150,7 +149,7 @@ def test_agnostic_ampl_PH(self): phoptions = _get_ph_base_options() phoptions["Ag"] = Ag # this is critical phoptions["solver_name"] = "gurobi" # need an ampl solver - scennames = farmer_ampl_agnostic.scenario_names_creator(num_scens=3) + scennames = scenario_names_creator(3) ph = mpisppy.opt.ph.PH( phoptions, scennames, @@ -224,7 +223,7 @@ def test_agnostic_gurobipy_PH_constructor(self): phoptions = _get_ph_base_options() ph = mpisppy.opt.ph.PH( phoptions, - farmer_gurobipy_agnostic.scenario_names_creator(num_scens=3), + scenario_names_creator(3), Ag.scenario_creator, farmer_gurobipy_agnostic.scenario_denouement, scenario_creator_kwargs=None, # agnostic.py takes care of this @@ -240,7 +239,7 @@ def test_agnostic_gurobipy_PH(self): s1 = Ag.scenario_creator("scen1") # average case phoptions = _get_ph_base_options() phoptions["Ag"] = Ag # this is critical - scennames = farmer_gurobipy_agnostic.scenario_names_creator(num_scens=3) + scennames = scenario_names_creator(3) ph = mpisppy.opt.ph.PH( phoptions, scennames, diff --git a/mpisppy/tests/test_conf_int_aircond.py b/mpisppy/tests/test_conf_int_aircond.py index 6e5c741e0..e73e147ef 100644 --- a/mpisppy/tests/test_conf_int_aircond.py +++ b/mpisppy/tests/test_conf_int_aircond.py @@ -18,17 +18,15 @@ from mpisppy.tests.utils import get_solver, round_pos_sig -import mpisppy.utils.sputils as sputils +from mpisppy.utils import amalgamator as ama, config, scenario_names_creator, sputils import mpisppy.tests.examples.aircond as aircond import mpisppy.confidence_intervals.mmw_ci as MMWci -import mpisppy.utils.amalgamator as ama from mpisppy.utils.xhat_eval import Xhat_Eval import mpisppy.confidence_intervals.seqsampling as seqsampling import mpisppy.confidence_intervals.multi_seqsampling as multi_seqsampling import mpisppy.confidence_intervals.confidence_config as confidence_config import mpisppy.confidence_intervals.ciutils as ciutils -from mpisppy.utils import config import pyomo.common.config as pyofig __version__ = 0.4 @@ -46,7 +44,7 @@ class Test_confint_aircond(unittest.TestCase): def setUpClass(self): self.refmodelname ="mpisppy.tests.examples.aircond" # amalgamator compatible # TBD: maybe this code should create the file - self.xhatpath = "farmer_cyl_nonants.spy.npy" + self.xhatpath = "farmer_cyl_nonants.spy.npy" def _get_base_options(self): # Base option has batch options @@ -79,7 +77,7 @@ def _get_xhatEval_options(self): def _get_xhat_gen_options(self, BFs): num_scens = np.prod(BFs) - scenario_names = aircond.scenario_names_creator(num_scens) + scenario_names = scenario_names_creator(num_scens) xhat_gen_options = {"scenario_names": scenario_names, "solver_name": solver_name, "solver_options": None, @@ -107,14 +105,14 @@ def setUp(self): def tearDown(self): os.remove(self.xhat_path) - + def _eval_creator(self): xh_options = self._get_xhatEval_options() MMW_options, scenario_creator_kwargs = self._get_base_options() branching_factors= global_BFs - scen_count = np.prod(branching_factors) - all_scenario_names = aircond.scenario_names_creator(scen_count) + scen_count = int(np.prod(branching_factors)) + all_scenario_names = scenario_names_creator(scen_count) all_nodenames = sputils.create_nodenames_from_branching_factors(branching_factors) ev = Xhat_Eval(xh_options, all_scenario_names, @@ -233,7 +231,7 @@ def test_xhat_eval_evaluate_one(self): num_scens = np.prod(branching_factors) scenario_creator_kwargs['num_scens'] = num_scens - all_scenario_names = aircond.scenario_names_creator(num_scens) + all_scenario_names = scenario_names_creator(num_scens) k = all_scenario_names[0] obj = ev.evaluate_one(full_xhat,k,ev.local_scenarios[k]) @@ -262,8 +260,8 @@ def test_MMW_running(self): def test_gap_estimators(self): options, scenario_creator_kwargs = self._get_base_options() branching_factors= global_BFs - scen_count = np.prod(branching_factors) - scenario_names = aircond.scenario_names_creator(scen_count, start=1000) + scen_count = int(np.prod(branching_factors)) + scenario_names = scenario_names_creator(scen_count, start=1000) sample_options = {"seed": 0, "branching_factors": global_BFs} estim = ciutils.gap_estimators(self.xhat, diff --git a/mpisppy/tests/test_conf_int_farmer.py b/mpisppy/tests/test_conf_int_farmer.py index e130b9277..56fa05aba 100644 --- a/mpisppy/tests/test_conf_int_farmer.py +++ b/mpisppy/tests/test_conf_int_farmer.py @@ -27,7 +27,7 @@ from mpisppy.utils.xhat_eval import Xhat_Eval import mpisppy.confidence_intervals.seqsampling as seqsampling import mpisppy.confidence_intervals.ciutils as ciutils -from mpisppy.utils import config +from mpisppy.utils import config, scenario_names_creator import mpisppy.confidence_intervals.confidence_config as confidence_config fullcomm = mpi.COMM_WORLD @@ -153,8 +153,7 @@ def test_ama_running(self): self.assertEqual(obj, -130000) - @unittest.skipIf(not solver_available, - "no solver is available") + @unittest.skipIf(not solver_available, "no solver is available") def test_xhat_eval_creator(self): options = self._get_xhatEval_options() @@ -162,22 +161,21 @@ def test_xhat_eval_creator(self): scenario_creator_kwargs = MMW_options['kwargs'] scenario_creator_kwargs['num_scens'] = MMW_options['batch_size'] ev = Xhat_Eval(options, - farmer.scenario_names_creator(100), + scenario_names_creator(100), farmer.scenario_creator, scenario_denouement=None, scenario_creator_kwargs=scenario_creator_kwargs ) assert ev is not None - @unittest.skipIf(not solver_available, - "no solver is available") + @unittest.skipIf(not solver_available, "no solver is available") def test_xhat_eval_evaluate(self): options = self._get_xhatEval_options() MMW_options = self._get_base_options() scenario_creator_kwargs = MMW_options['kwargs'] scenario_creator_kwargs['num_scens'] = MMW_options['batch_size'] ev = Xhat_Eval(options, - farmer.scenario_names_creator(100), + scenario_names_creator(100), farmer.scenario_creator, scenario_denouement=None, scenario_creator_kwargs=scenario_creator_kwargs @@ -187,15 +185,14 @@ def test_xhat_eval_evaluate(self): obj = round_pos_sig(ev.evaluate(xhat),2) self.assertEqual(obj, -1300000.0) - @unittest.skipIf(not solver_available, - "no solver is available") + @unittest.skipIf(not solver_available, "no solver is available") def test_xhat_eval_evaluate_one(self): options = self._get_xhatEval_options() MMW_options = self._get_base_options() xhat = ciutils.read_xhat(self.xhat_path) scenario_creator_kwargs = MMW_options['kwargs'] scenario_creator_kwargs['num_scens'] = MMW_options['batch_size'] - scenario_names = farmer.scenario_names_creator(100) + scenario_names = scenario_names_creator(100) ev = Xhat_Eval(options, scenario_names, farmer.scenario_creator, @@ -207,8 +204,7 @@ def test_xhat_eval_evaluate_one(self): obj = round_pos_sig(obj,2) self.assertEqual(obj, -48000.0) - @unittest.skipIf(not solver_available, - "no solver is available") + @unittest.skipIf(not solver_available, "no solver is available") def test_MMW_running(self): cfg = self._get_base_options() xhat = ciutils.read_xhat(self.xhat_path) @@ -223,10 +219,9 @@ def test_MMW_running(self): bound = round_pos_sig(r['gap_inner_bound'],2) self.assertEqual((s,bound), (1.5,96.0)) - @unittest.skipIf(not solver_available, - "no solver is available") + @unittest.skipIf(not solver_available, "no solver is available") def test_gap_estimators(self): - scenario_names = farmer.scenario_names_creator(50,start=1000) + scenario_names = scenario_names_creator(50, start=1000) estim = ciutils.gap_estimators(self.xhat, self.refmodelname, cfg=self._get_base_options(), diff --git a/mpisppy/tests/test_gradient_rho.py b/mpisppy/tests/test_gradient_rho.py index f615acdf3..01c025ae2 100644 --- a/mpisppy/tests/test_gradient_rho.py +++ b/mpisppy/tests/test_gradient_rho.py @@ -16,7 +16,7 @@ import unittest from mpisppy.utils import config -import mpisppy.utils.cfg_vanilla as vanilla +from mpisppy.utils import cfg_vanilla as vanilla, scenario_names_creator import mpisppy.tests.examples.farmer as farmer from mpisppy.spin_the_wheel import WheelSpinner from mpisppy.tests.utils import get_solver @@ -49,11 +49,11 @@ def _create_ph_farmer(self): self.cfg.num_scens = 3 scenario_creator = farmer.scenario_creator scenario_denouement = farmer.scenario_denouement - all_scenario_names = farmer.scenario_names_creator(self.cfg.num_scens) + all_scenario_names = scenario_names_creator(self.cfg.num_scens, prefix="scen") scenario_creator_kwargs = farmer.kw_creator(self.cfg) beans = (self.cfg, scenario_creator, scenario_denouement, all_scenario_names) hub_dict = vanilla.ph_hub(*beans, scenario_creator_kwargs=scenario_creator_kwargs) - hub_dict['opt_kwargs']['options']['cfg'] = self.cfg + hub_dict['opt_kwargs']['options']['cfg'] = self.cfg list_of_spoke_dict = list() wheel = WheelSpinner(hub_dict, list_of_spoke_dict) wheel.spin() diff --git a/mpisppy/tests/test_scenario_names_creator.py b/mpisppy/tests/test_scenario_names_creator.py new file mode 100644 index 000000000..e0f05455e --- /dev/null +++ b/mpisppy/tests/test_scenario_names_creator.py @@ -0,0 +1,43 @@ +############################################################################### +# mpi-sppy: MPI-based Stochastic Programming in PYthon +# +# Copyright (c) 2024, Lawrence Livermore National Security, LLC, Alliance for +# Sustainable Energy, LLC, The Regents of the University of California, et al. +# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md for +# full copyright and license information. +############################################################################### + +from unittest import TestCase + +from mpisppy.utils import scenario_names_creator + + +class TestScenarioNamesCeator(TestCase): + def test_scenario_names_creator_default(self): + self.assertEqual( + scenario_names_creator(3), + ["scenario0", "scenario1", "scenario2"], + "Should be a list with 'scenario0', 'scenario1' and 'scenario2'", + ) + + def test_scenario_names_creator_defining_prefix(self): + self.assertEqual( + scenario_names_creator(3, prefix="s"), + ["s0", "s1", "s2"], + "Should be a list with 's0', 's1' and 's2'", + ) + + def test_scenario_names_creator_setting_start_value(self): + self.assertEqual( + scenario_names_creator(3, start=1), + ["scenario1", "scenario2", "scenario3"], + "Should be a list with 'scenario1', 'scenario2' and 'scenario3'", + ) + + def test_scenario_names_creator_start_value_is_None(self): + # to avoid migration errors from earlier implementations start is also allowed to be None + self.assertEqual( + scenario_names_creator(3, start=None), + ["scenario0", "scenario1", "scenario2"], + "Should be a list with 'scenario10', 'scenario1' and 'scenario2'", + ) diff --git a/mpisppy/tests/test_with_cylinders.py b/mpisppy/tests/test_with_cylinders.py index 7b5c45cd9..917268a71 100644 --- a/mpisppy/tests/test_with_cylinders.py +++ b/mpisppy/tests/test_with_cylinders.py @@ -13,9 +13,8 @@ """ import unittest -from mpisppy.utils import config +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator -import mpisppy.utils.cfg_vanilla as vanilla import mpisppy.tests.examples.farmer as farmer from mpisppy.spin_the_wheel import WheelSpinner from mpisppy.tests.utils import get_solver @@ -50,7 +49,7 @@ def _create_stuff(self, iters=5): self.cfg.max_iterations = iters scenario_creator = farmer.scenario_creator scenario_denouement = farmer.scenario_denouement - all_scenario_names = farmer.scenario_names_creator(self.cfg.num_scens) + all_scenario_names = scenario_names_creator(self.cfg.num_scens) scenario_creator_kwargs = farmer.kw_creator(self.cfg) beans = (self.cfg, scenario_creator, scenario_denouement, all_scenario_names) hub_dict = vanilla.ph_hub(*beans, scenario_creator_kwargs=scenario_creator_kwargs) diff --git a/mpisppy/tests/test_xbar_w_reader_writer.py b/mpisppy/tests/test_xbar_w_reader_writer.py index c16037750..e3cb61030 100644 --- a/mpisppy/tests/test_xbar_w_reader_writer.py +++ b/mpisppy/tests/test_xbar_w_reader_writer.py @@ -16,14 +16,12 @@ import os import unittest import csv -from mpisppy.utils import config +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator, wxbarreader, wxbarwriter -import mpisppy.utils.cfg_vanilla as vanilla import mpisppy.tests.examples.farmer as farmer from mpisppy.spin_the_wheel import WheelSpinner from mpisppy.tests.utils import get_solver -import mpisppy.utils.wxbarreader as wxbarreader -import mpisppy.utils.wxbarwriter as wxbarwriter + __version__ = 0.2 @@ -31,7 +29,7 @@ def _create_cfg(): cfg = config.Config() - wxbarreader.add_options_to_config(cfg) + wxbarreader.add_options_to_config(cfg) wxbarwriter.add_options_to_config(cfg) cfg.add_branching_factors() cfg.num_scens_required() @@ -55,7 +53,7 @@ def _create_ph_farmer(self, ph_extensions=None, max_iter=100): self.cfg.num_scens = 3 scenario_creator = farmer.scenario_creator scenario_denouement = farmer.scenario_denouement - all_scenario_names = farmer.scenario_names_creator(self.cfg.num_scens) + all_scenario_names = scenario_names_creator(self.cfg.num_scens, prefix="scen") scenario_creator_kwargs = farmer.kw_creator(self.cfg) self.cfg.max_iterations = max_iter beans = (self.cfg, scenario_creator, scenario_denouement, all_scenario_names) diff --git a/mpisppy/utils/__init__.py b/mpisppy/utils/__init__.py index 776b873ec..480a79e89 100644 --- a/mpisppy/utils/__init__.py +++ b/mpisppy/utils/__init__.py @@ -7,7 +7,7 @@ # full copyright and license information. ############################################################################### -from .strings import nice_join +from .strings import nice_join, scenario_names_creator -__all__ = ["nice_join"] +__all__ = ["nice_join", "scenario_names_creator"] diff --git a/mpisppy/utils/amalgamator.py b/mpisppy/utils/amalgamator.py index 7fc6f6b74..bd77d9ba5 100644 --- a/mpisppy/utils/amalgamator.py +++ b/mpisppy/utils/amalgamator.py @@ -13,12 +13,9 @@ import pyomo.environ as pyo import copy import pyomo.common.config as pyofig -from mpisppy.utils import config -import mpisppy.utils.solver_spec as solver_spec +from mpisppy.utils import cfg_vanilla as vanilla, config, scenario_names_creator, solver_spec, sputils from mpisppy.spin_the_wheel import WheelSpinner -import mpisppy.utils.sputils as sputils -import mpisppy.utils.cfg_vanilla as vanilla from mpisppy import global_toc from mpisppy.extensions.fixer import Fixer @@ -126,10 +123,7 @@ def find_spokes(cylinders, is_multi=False): #========== def check_module_ama(module): # Complain if the module lacks things needed. - everything = ["scenario_names_creator", - "scenario_creator", - "inparser_adder", - "kw_creator"] # start and denouement can be missing. + everything = ["scenario_creator", "inparser_adder", "kw_creator"] # start and denouement can be missing. you_can_have_it_all = True for ething in everything: if not hasattr(module, ething): @@ -169,7 +163,7 @@ def from_module(mname, cfg, extraargs_fct=None, use_command_line=True): use_command_line=use_command_line) cfg.add_and_assign('_mpisppy_probability', description="Uniform prob.", domain=float, default=None, value= 1/cfg['num_scens'], complain=False) start = cfg['start'] if 'start' in cfg else 0 - sn = m.scenario_names_creator(cfg['num_scens'], start=start) + sn = scenario_names_creator(cfg['num_scens'], start=start) dn = m.scenario_denouement if hasattr(m, "scenario_denouement") else None ama = Amalgamator(cfg, sn, diff --git a/mpisppy/utils/mps_module.py b/mpisppy/utils/mps_module.py index 1b5cbbc7a..93714d610 100644 --- a/mpisppy/utils/mps_module.py +++ b/mpisppy/utils/mps_module.py @@ -26,12 +26,9 @@ """ import os -import re -import glob import json import mpisppy.scenario_tree as scenario_tree -import mpisppy.utils.sputils as sputils -import mpisppy.utils.mps_reader as mps_reader +from mpisppy.utils import mps_reader, sputils # assume you can get the path from config, set in kw_creator as a side-effect mps_files_directory = None @@ -87,31 +84,6 @@ def scenario_creator(sname, cfg=None): return model -#========= -def scenario_names_creator(num_scens, start=None): - # validate the directory and use it to get names (that have to be numbered) - # IMPORTANT: start is zero-based even if the names are one-based! - mps_files = [os.path.basename(f) - for f in glob.glob(os.path.join(mps_files_directory, "*.mps"))] - mps_files.sort() - if start is None: - start = 0 - if num_scens is None: - num_scens = len(mps_files) - start - first = re.search(r"\d+$",mps_files[0][:-4]) # first scenario number - try: - first = int(first.group()) - except Exception as e: - raise RuntimeError(f'mps files in {mps_files_directory} must end with an integer' - f'found file {mps_files[0]} (error was: {e})') - if first != 0: - print("WARNING: non-zero-based senario names might cause trouble" - f" found {first=} for dir {mps_files_directory}") - assert start+num_scens <= len(mps_files),\ - f"Trying to create scenarios names with {start=}, {num_scens=} but {len(mps_files)=}" - retval = [fn[:-4] for fn in mps_files[start:start+num_scens]] - return retval - #========= def inparser_adder(cfg): # verify that that the mps_files_directory is there, or add it diff --git a/mpisppy/utils/proper_bundler.py b/mpisppy/utils/proper_bundler.py index 15612d92f..439a06336 100644 --- a/mpisppy/utils/proper_bundler.py +++ b/mpisppy/utils/proper_bundler.py @@ -11,8 +11,8 @@ import os import numpy as np -import mpisppy.utils.sputils as sputils -import mpisppy.utils.pickle_bundle as pickle_bundle +from mpisppy.utils import pickle_bundle, scenario_names_creator, sputils + # Development notes: # 2 stage Cases: @@ -53,7 +53,7 @@ def inparser_adder(self, cfg): def scenario_names_creator(self, num_scens, start=None, cfg=None): # no need to wrap? assert cfg is not None, "ProperBundler needs cfg for scenario names" - return cfg.model.scenario_names_creator(num_scens, start=start) + return scenario_names_creator(num_scens, start=start) def set_bunBFs(self, cfg): # utility for bundle objects. Might throw if it doesn't like the branching factors. @@ -82,7 +82,7 @@ def bundle_names_creator(self, num_buns, start=None, cfg=None): bsize = cfg.scenarios_per_bundle # typing aid self.set_bunBFs(cfg) # We need to know if scenarios (not bundles) are one-based. - inum = sputils.extract_num(self.module.scenario_names_creator(1)[0]) + inum = sputils.extract_num(scenario_names_creator(1)[0]) names = [f"Bundle_{bn*bsize+inum}_{(bn+1)*bsize-1+inum}" for bn in range(start+num_buns)] return names @@ -114,8 +114,7 @@ def scenario_creator(self, sname, **kwargs): firstnum = int(sname.split("_")[1]) # sname is a bundle name lastnum = int(sname.split("_")[2]) # snames are scenario names - snames = self.module.scenario_names_creator(lastnum-firstnum+1, - firstnum) + snames = scenario_names_creator(lastnum-firstnum+1, start=firstnum) kws = self.original_kwargs if self.bunBFs is not None: # The original scenario creator needs to handle these diff --git a/mpisppy/utils/sputils.py b/mpisppy/utils/sputils.py index c7bff5e45..2ef1ac488 100644 --- a/mpisppy/utils/sputils.py +++ b/mpisppy/utils/sputils.py @@ -28,6 +28,7 @@ from pyomo.core.base.indexed_component_slice import IndexedComponent_slice from mpisppy import tt_timer +from mpisppy.utils import scenario_names_creator global_rank = MPI.COMM_WORLD.Get_rank() @@ -729,7 +730,7 @@ def scens_to_ranks(scen_count, n_proc, rank, branching_factors = None): # indecision as of May 2020 (delete this comment DLW) # just make sure things are consistent with what xhat will do... # TBD: streamline - all_scenario_names = ["ID"+str(i) for i in range(scen_count)] + all_scenario_names = scenario_names_creator(scen_count, prefix="ID") tree = _ScenTree(branching_factors, all_scenario_names) scenario_names_to_ranks, slices, = tree.scen_name_to_rank(n_proc, rank) return slices, scenario_names_to_ranks diff --git a/mpisppy/utils/strings.py b/mpisppy/utils/strings.py index e69e2d49b..fdb16c3cd 100644 --- a/mpisppy/utils/strings.py +++ b/mpisppy/utils/strings.py @@ -45,3 +45,25 @@ def nice_join( return separator.join(seq) else: return f"{separator.join(seq[:-1])} {conjunction} {seq[-1]}" + + +def scenario_names_creator(n: int, prefix: str = "scenario", start: int | None = 0) -> list[str]: + """Creates a list of scenario names using the pattern `{prefix}{number}` for a single scenario + name. The scenario names are distinguished using consecutive numbers. + + Args: + n (int): number of wanted scenarios names + prefix (str, "scenario"): string to start the scenario name + start (int | None, 0): number to use for the first scenario name + + Examples: + + >>> from mpisppy.utils import scenario_names_creator + + >>> scenario_names_creator(3, prefix="scen_", start=1) + ["scen_1", "scen_2", "scen_3"] + """ + # start is also allowed to be None to avoid migration errors from earlier implementations + if start is None: + start = 0 + return [f"{prefix}{i}" for i in range(start, start + n)] diff --git a/mpisppy/utils/wtracker.py b/mpisppy/utils/wtracker.py index 04898ebb9..9e4f55da0 100644 --- a/mpisppy/utils/wtracker.py +++ b/mpisppy/utils/wtracker.py @@ -20,6 +20,7 @@ import pandas as pd import mpisppy.opt.ph import mpisppy.MPI as MPI +from mpisppy.utils import scenario_names_creator class WTracker(): """ @@ -233,7 +234,7 @@ def check_w_stdev(self, wlen, stdevthresh, offsetback=0): options["PHIterLimit"] = 1 ph = mpisppy.opt.ph.PH( options, - farmer.scenario_names_creator(3), + scenario_names_creator(3), farmer.scenario_creator, farmer.scenario_denouement, scenario_creator_kwargs=farmer.kw_creator(options), diff --git a/paperruns/larger_uc/uc_cylinders.py b/paperruns/larger_uc/uc_cylinders.py index ef8cc7e31..0c87d6aa1 100644 --- a/paperruns/larger_uc/uc_cylinders.py +++ b/paperruns/larger_uc/uc_cylinders.py @@ -20,8 +20,7 @@ from mpisppy.extensions.extension import MultiExtension from mpisppy.extensions.fixer import Fixer from mpisppy.extensions.mipgapper import Gapper -from mpisppy.utils import baseparsers -from mpisppy.utils import vanilla +from mpisppy.utils import baseparsers, cfg_vanilla as vanilla, scenario_names_creator from mpisppy.extensions.cross_scen_extension import CrossScenarioExtension @@ -38,7 +37,7 @@ def _parse_args(): help="json file with mipgap schedule (default None)", dest="ph_mipgaps_json", type=str, - default=None) + default=None) args = parser.parse_args() return args @@ -69,7 +68,7 @@ def main(): } scenario_creator = uc.scenario_creator scenario_denouement = uc.scenario_denouement - all_scenario_names = [f"Scenario{i+1}" for i in range(num_scen)] + all_scenario_names = scenario_names_creator(num_scen, prefix="Scenario", start=1) rho_setter = uc._rho_setter # Things needed for vanilla cylinders