diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 11bad7b32..4159ddb14 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,6 +1,24 @@ Next release ============ +Migration notes +--------------- + +Adjust any imports like the following: + +.. code-block:: python + + from message_ix.models import DIMS, Item, MACRO, MESSAGE, MESSAGE_MACRO + +…to: + +.. code-block:: python + + from message_ix.common import DIMS, Item + from message_ix.macro import MACRO + from message_ix.message import MESSAGE + from message_ix.message_macro import MESSAGE_MACRO + All changes ----------- @@ -20,6 +38,13 @@ All changes - Revise :ref:`equation_commodity_balance_aux` to include input and output flows based on |CAP| and |CAP_NEW|. - New :class:`.MESSAGE` / :meth:`.Scenario.solve` option :py:`cap_comm=True` to enable this representation. +- The former module :py:`message_ix.models` is split to distinct submodules (:pull:`972`): + + - :mod:`message_ix.common` includes :class:`.GAMSModel` and related code. + - :mod:`message_ix.macro` includes :class:`.MACRO`. + - :mod:`message_ix.message` includes :class:`.MESSAGE`. + - :mod:`message_ix.message_macro` includes :class:`.MESSAGE_MACRO`. + - Document the :ref:`minimum version of Java ` required for :class:`ixmp.JDBCBackend ` (:pull:`962`). - Improve type hinting (:pull:`963`). @@ -61,7 +86,7 @@ Users **should**: All changes ----------- -- Some MESSAGEix :doc:`tutorials ` are runnable with the :class:`.IXMP4Backend` introduced in :mod:`ixmp` version 3.11 (:pull:`894`, :pull:`941`). +- Some MESSAGEix :doc:`tutorials ` are runnable with the :class:`~ixmp.backend.ixmp4.IXMP4Backend` introduced in :mod:`ixmp` version 3.11 (:pull:`894`, :pull:`941`). See `Support roadmap for ixmp4 `__ for details. - Add the :py:`concurrent=...` model option to :class:`.MACRO` (:pull:`808`). - Adjust use of :ref:`type_tec ` in :ref:`equation_emission_equivalence` (:pull:`930`, :issue:`929`, :pull:`935`). @@ -174,7 +199,7 @@ Migration notes NOTE: this may result in changes to the solution. In order to use the previous default `lpmethod`, the user-specific default setting can be set through the user's ixmp configuration file. Alternatively, the `lpmethod` can be specified directly as an argument when solving a scenario. - Both of these configuration methods are further explained :meth:`here `. + Both of these configuration methods are further documented at :class:`.GAMSModel`. - The dimensionality of one set and two parameters (``map_tec_storage``, ``storage_initial``, and ``storage_self_discharge``) are extended to allow repesentation of the mode of operation of storage technologies and the temporal level of storage containers. If these items are already populated with data in a Scenario, this data will be incompatible with the MESSAGE GAMS implementation in this release; a :class:`UserWarning` will be emitted when the :class:`.Scenario` is instantiated, and :meth:`~.message_ix.Scenario.solve` will raise a :class:`ValueError`. @@ -421,7 +446,7 @@ All changes - :pull:`281`: Test and improve logic of :meth:`.years_active` and :meth:`.vintage_and_active_years`. - :pull:`269`: Enforce ``year``-indexed columns as integers. - :pull:`256`: Update to use :obj:`ixmp.config` and improve CLI. -- :pull:`255`: Add :mod:`message_ix.testing.nightly` and ``message-ix nightly`` CLI command group for slow-running tests. +- :pull:`255`: Add :py:`message_ix.testing.nightly` and ``message-ix nightly`` CLI command group for slow-running tests. - :pull:`249`, :pull:`259`: Build MESSAGE and MESSAGE_MACRO classes on ixmp model API; adjust Scenario. - :pull:`235`: Add a reporting tutorial. - :pull:`236`, :pull:`242`, :pull:`263`: Enhance reporting. diff --git a/doc/api.rst b/doc/api.rst index 71c851c20..f504f1061 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -129,17 +129,21 @@ The full API is also available from R; see :doc:`rmessageix`. Model classes ------------- -.. currentmodule:: message_ix.models +.. currentmodule:: message_ix .. autosummary:: - MESSAGE - MACRO - MESSAGE_MACRO - GAMSModel - DEFAULT_CPLEX_OPTIONS - Item - ItemType + ~message.MESSAGE + ~macro.MACRO + ~message_macro.MESSAGE_MACRO + ~common.GAMSModel + ~common.DEFAULT_CPLEX_OPTIONS + ~common.Item + ~ixmp.backend.ItemType + +.. currentmodule:: message_ix.common + +.. automodule:: message_ix.common .. autodata:: DEFAULT_CPLEX_OPTIONS @@ -149,7 +153,8 @@ Model classes :members: :exclude-members: defaults - The :class:`.MESSAGE`, :class:`MACRO`, and :class:`MESSAGE_MACRO` child classes encapsulate the GAMS code for the core MESSAGE (or MACRO) mathematical formulation. + The :class:`.MESSAGE`, :class:`.MACRO`, and :class:`.MESSAGE_MACRO` child classes + encapsulate the GAMS code for the core MESSAGE (or MACRO) mathematical formulation. The class receives `model_options` via :meth:`.Scenario.solve`. Some of these are passed on to the parent class :class:`ixmp.model.gams.GAMSModel` (see there for a list); others are handled as described below. @@ -244,6 +249,10 @@ Model classes * - **var_list** - :obj:`None` +.. currentmodule:: message_ix.message + +.. automodule:: message_ix.message + .. autoclass:: MESSAGE :members: initialize :exclude-members: defaults @@ -267,6 +276,10 @@ Model classes Keys are the names of items (sets, parameters, variables, and equations); values are :class:`.Item` instances. These include all items listed in the MESSAGE mathematical specification, i.e. :ref:`sets_maps_def` and :ref:`parameter_def`. +.. currentmodule:: message_ix.macro + +.. automodule:: message_ix.macro + .. autoclass:: MACRO :members: :exclude-members: items @@ -283,6 +296,10 @@ Model classes .. autoattribute:: items :no-value: +.. currentmodule:: message_ix.message_macro + +.. automodule:: message_ix.message_macro + .. autoclass:: MESSAGE_MACRO :members: :exclude-members: items @@ -307,12 +324,10 @@ Model classes .. autoattribute:: items :no-value: -.. autodata:: DIMS -.. autoclass:: Item +.. autodata:: message_ix.common.DIMS +.. autoclass:: message_ix.common.Item :members: -.. currentmodule:: message_ix.macro - .. _utils: Utility methods @@ -321,14 +336,8 @@ Utility methods .. automodule:: message_ix.util :members: expand_dims, copy_model, make_df -.. automodule:: message_ix.util.sankey - :members: map_for_sankey - Testing utilities ----------------- .. automodule:: message_ix.testing :members: make_austria, make_dantzig, make_westeros, tmp_model_dir - -.. automodule:: message_ix.testing.nightly - :members: diff --git a/doc/conf.py b/doc/conf.py index aa19ca136..2fc56a29f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -165,6 +165,7 @@ # top-level module in objects.inv. Resolve these using :doc:`index` or similar for # each project. "dask$": ":std:doc:`dask:index`", + "jpype$": ":std:doc:`jpype:index`", "plotnine$": ":class:`plotnine.ggplot`", } diff --git a/doc/install-adv.rst b/doc/install-adv.rst index a41171cae..1ef51027f 100644 --- a/doc/install-adv.rst +++ b/doc/install-adv.rst @@ -230,7 +230,7 @@ This implies five of the six available groups of extra requirements: - ``docs`` includes packages required to build this documentation locally, including ``message_ix[report]`` and all *its* requirements, -- ``ixmp4`` includes packages require to use :class:`ixmp.IXMP4Backend <.IXMP4Backend>`, +- ``ixmp4`` includes packages require to use :class:`ixmp.IXMP4Backend `, - ``report`` includes packages required to use the built-in :doc:`reporting ` features of :mod:`message_ix`, - ``sankey`` includes packages required to use :meth:`.Reporter.add_sankey`, - ``tests`` includes packages required to run the test suite, diff --git a/doc/macro.rst b/doc/macro.rst index 88b994a54..c64d1bd3f 100644 --- a/doc/macro.rst +++ b/doc/macro.rst @@ -73,10 +73,12 @@ The required dimensions or symbol of each item are given in the same notation us - ``price_ref`` (:math:`n, s`): prices of MACRO sector output in the reference year. These can be constructed from the MESSAGE variable ``PRICE_COMMODITY``, using the ``config`` mapping. - If not provided, :mod:`message_ix.macro` will identify the reference year and extrapolate reference values using an exponential function fitted to ``PRICE_COMMODITY`` values; see :func:`.macro.extrapolate`. + If not provided, :mod:`message_ix.macro.calibrate` will identify the reference year + and extrapolate reference values using an exponential function fitted to ``PRICE_COMMODITY`` values. + See :func:`~.macro.calibrate.extrapolate`. - ``cost_ref`` (:math:`n`): total cost of the energy system in the reference year. These can be constructed from the MESSAGE variable ``COST_NODAL_NET``, including dividing by a factor of 1000. - If not provided, :mod:`message_ix.macro` will extrapolate using :func:`.macro.extrapolate`. + If not provided, :mod:`message_ix.macro.calibrate` will extrapolate using :func:`~.macro.calibrate.extrapolate`. - ``demand_ref`` (:math:`n, s`): demand for MACRO sector output in the reference year. - ``lotol`` (:math:`n`): tolerance factor for lower bounds on MACRO variables. - ``esub`` (:math:`\epsilon_n`): elasticity of substitution between capital-labor and energy. @@ -228,9 +230,9 @@ Alternatively the arguments can be specified either in :file:`models.py`. Code documentation ================== -.. currentmodule:: message_ix.macro +.. currentmodule:: message_ix.macro.calibrate -.. automodule:: message_ix.macro +.. automodule:: message_ix.macro.calibrate :members: The functions :func:`add_model_data` and :func:`calibrate` are used by :meth:`.Scenario.add_macro`. diff --git a/doc/reporting.rst b/doc/reporting.rst index 540f20952..8bf42a042 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -179,7 +179,7 @@ These include: Other added keys include: -- :mod:`message_ix` adds the standard short symbols for |MESSAGEix| dimensions (sets) based on :data:`.models.DIMS`. +- :mod:`message_ix` adds the standard short symbols for |MESSAGEix| dimensions (sets) based on :data:`.common.DIMS`. Each of these is also available in a Reporter: for example :py:`rep.get("n")` returns a list with the elements of the |MESSAGEix| set named "node"; :py:`rep.get("t")` returns the elements of the set "technology", and so on. These keys can be used as input to other computations. diff --git a/message_ix/__init__.py b/message_ix/__init__.py index 0b001c37e..cf5770c68 100644 --- a/message_ix/__init__.py +++ b/message_ix/__init__.py @@ -8,7 +8,9 @@ from ixmp.util import DeprecatedPathFinder from .core import Scenario -from .models import MACRO, MESSAGE, MESSAGE_MACRO +from .macro import MACRO +from .message import MESSAGE +from .message_macro import MESSAGE_MACRO from .report import Reporter from .util import make_df diff --git a/message_ix/common.py b/message_ix/common.py new file mode 100644 index 000000000..3ec1abe3a --- /dev/null +++ b/message_ix/common.py @@ -0,0 +1,300 @@ +import logging +import re +from collections import ChainMap +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from copy import copy +from dataclasses import InitVar, dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + +import ixmp.model.gams +from ixmp import config +from ixmp.backend import ItemType + +if TYPE_CHECKING: + from logging import LogRecord + + from ixmp.types import InitializeItemsKwargs + + +log = logging.getLogger(__name__) + +#: Solver options used by :meth:`.Scenario.solve`. +DEFAULT_CPLEX_OPTIONS = { + "advind": 0, + "lpmethod": 4, + "threads": 4, + "epopt": 1e-6, +} + +#: Common dimension name abbreviations mapped to tuples with: +#: +#: 1. the respective coordinate/index set, and +#: 2. the full dimension name. +DIMS = { + "c": ("commodity", "commodity"), + "c2": ("commodity", "commodity2"), + "e": ("emission", "emission"), + "first_period": ("year", "first_period"), + "g": ("grade", "grade"), + "h": ("time", "time"), + "h2": ("time", "time2"), + "hd": ("time", "time_dest"), + "ho": ("time", "time_origin"), + "inv_tec": ("technology", "inv_tec"), + "l": ("level", "level"), + "m": ("mode", "mode"), + "ms": ("mode", "storage_mode"), + "n": ("node", "node"), + "nd": ("node", "node_dest"), + "nl": ("node", "node_loc"), + "no": ("node", "node_origin"), + "node_parent": ("node", "node_parent"), + "nr": ("node", "node_rel"), + "ns": ("node", "node_share"), + "q": ("rating", "rating"), + "r": ("relation", "relation"), + "renewable_tec": ("technology", "renewable_tec"), + "s": ("land_scenario", "land_scenario"), + "t": ("technology", "technology"), + "ta": ("technology", "technology_addon"), + "time_parent": ("time", "time_parent"), + "tp": ("technology", "technology_primary"), + "ts": ("technology", "storage_tec"), + "u": ("land_type", "land_type"), + "y": ("year", "year"), + "ya": ("year", "year_act"), + "yr": ("year", "year_rel"), + "yv": ("year", "year_vtg"), +} + + +@dataclass +class Item: + """Description of an :mod:`ixmp` item: equation, parameter, set, or variable. + + Instances of this class carry only structural information, not data. + """ + + #: Name of the item. + name: str + + #: Type of the item, for instance :attr:`ItemType.PAR `. + type: "ixmp.backend.ItemType" + + #: String expression for :attr:`coords` and :attr:`dims`. Split on spaces and parsed + #: using :data:`DIMS` so that, for instance, "nl yv" results in entries for + #: for "node", "year" in :attr:`coords`, and "node_loc", "year_vtg" in :attr:`dims`. + expr: InitVar[str] = "" + + #: Coordinates of the item; that is, the names of sets that index its dimensions. + #: The same set name may be repeated if it indexes multiple dimensions. + coords: tuple[str, ...] = field(default_factory=tuple) + + #: Dimensions of the item. + dims: tuple[str, ...] = field(default_factory=tuple) + + #: Text description of the item. + description: Optional[str] = None + + def __post_init__(self, expr): + if expr == "": + return + + # Split on spaces. For each dimension, use an abbreviation (if one) exists, else + # the set name for both coords and dims + self.coords, self.dims = zip(*[DIMS.get(d, (d, d)) for d in expr.split()]) + + if self.dims == self.coords: + # No distinct dimension names; don't store these + self.dims = tuple() + + @property + def ix_type(self) -> str: + """Lower-case string form of :attr:`type`: "equ", "par", "set", or "var". + + Read-only. + """ + return str(self.type.name).lower() + + def to_dict(self) -> "InitializeItemsKwargs": + """Return the :class:`dict` representation used internally in :mod:`ixmp`.""" + result: "InitializeItemsKwargs" = dict( + ix_type=self.ix_type, idx_sets=self.coords + ) + if self.dims: + result["idx_names"] = self.dims + return result + + +def _template(*parts): + """Helper to make a template string relative to model_dir.""" + return str(Path("{model_dir}", *parts)) + + +class GAMSModel(ixmp.model.gams.GAMSModel): + """Extended :class:`ixmp.model.gams.GAMSModel` for MESSAGE & MACRO.""" + + #: Default model options. + defaults = ChainMap( + { + # New keys for MESSAGE & MACRO + "model_dir": Path(__file__).parent / "model", + # Override keys from GAMSModel + "model_file": _template("{model_name}_run.gms"), + "in_file": _template("data", "MsgData_{case}.gdx"), + "out_file": _template("output", "MsgOutput_{case}.gdx"), + "solve_args": [ + '--in="{in_file}"', + '--out="{out_file}"', + '--iter="{}"'.format( + _template("output", "MsgIterationReport_{case}.gdx") + ), + ], + # Disable the feature to put input/output GDX files, list files, etc. in a + # temporary directory + "use_temp_dir": False, + # Record versions of message_ix and ixmp in GDX I/O files + "record_version_packages": ("message_ix", "ixmp"), + }, + ixmp.model.gams.GAMSModel.defaults, + ) + + #: Mapping from model item (equation, parameter, set, or variable) names to + #: :class:`.Item` describing the item. + items: Mapping[str, Item] + + # Make default model options known to the class + model_dir: Path + + #: Optional minimum version of GAMS. + GAMS_min_version: Optional[str] = None + + #: Keyword arguments to map to GAMS `solve_args`. + keyword_to_solve_arg: list[tuple[str, type, str]] + + def __init__(self, name: Optional[str] = None, **model_options) -> None: + if gmv := self.GAMS_min_version: + # Check the minimum GAMS version. + version = ixmp.model.gams.gams_version() or "" + if version < gmv: + raise RuntimeError(f"{self.name} requires GAMS >= {gmv}; got {version}") + + # Convert optional `keyword` arguments to GAMS CLI arguments like ``--{target}`` + solve_args = [] + for keyword, callback, target in self.keyword_to_solve_arg: + try: + raw = model_options.pop(keyword) + solve_args.append(f"--{target}={callback(raw)!s}") + except KeyError: + pass + except ValueError: + raise ValueError(f"{keyword} = {raw}") + + # Update the default options with any user-provided options + model_options.setdefault("model_dir", config.get("message model dir")) + self.cplex_opts = copy(DEFAULT_CPLEX_OPTIONS) + self.cplex_opts.update(config.get("message solve options") or dict()) + self.cplex_opts.update(model_options.pop("solve_options", {})) + + super().__init__(name, **model_options) + + self.solve_args.extend(solve_args) + + def run(self, scenario: "ixmp.Scenario") -> None: + """Execute the model. + + GAMSModel creates a file named ``cplex.opt`` in the model directory containing + the “solve_options”, as described above. + + .. warning:: GAMSModel can solve Scenarios in two or more Python processes + simultaneously; but using *different* CPLEX options in each process may + produce unexpected results. + """ + # Ensure the data in `scenario` is consistent with the MESSAGE formulation + self.enforce(scenario) + + # If two runs are kicked off simultaneously with the same self.model_dir, then + # they will try to write the same optfile, and may write different contents. + # + # TODO Re-enable the 'use_temp_dir' feature from ixmp.GAMSModel (disabled above) + # so that cplex.opt will be specific to that directory. + + # Write CPLEX options into an options file + optfile = Path(self.model_dir).joinpath("cplex.opt") + lines = ("{} = {}".format(*kv) for kv in self.cplex_opts.items()) + optfile.write_text("\n".join(lines)) + log.info(f"Use CPLEX options {self.cplex_opts}") + + self.cplex_opts.update({"barcrossalg": 2}) + optfile2 = Path(self.model_dir).joinpath("cplex.op2") + lines2 = ("{} = {}".format(*kv) for kv in self.cplex_opts.items()) + optfile2.write_text("\n".join(lines2)) + + result = super().run(scenario) + + # In previous versions, the `cplex.opt` file(s) were removed at this point + # in the workflow. This has been removed due to issues when running + # scenarios asynchronously. + + return result + + +class _boollike(str): + """Handle a :class:`bool`-like argument; return :py:`"0"` or :py:`"1"`.""" + + def __new__(cls, value: Any): + if value in {"0", 0, False, "False"}: + return "0" + elif value in {"1", 1, True, "True"}: + return "1" + else: + raise ValueError + + +@contextmanager +def _filter_log_initialize_items(cls: type[GAMSModel]) -> Iterator[None]: + """Context manager to filter log messages related to existing items.""" + pattern = _log_filter_pattern(cls) + + def _filter(record: "LogRecord") -> bool: + # Retrieve the compiled expression + return not pattern.match(record.msg) + + # Attach the filter to the ixmp.model logger + logger = logging.getLogger("ixmp.model.base") + logger.addFilter(_filter) + try: + yield + finally: + logger.removeFilter(_filter) + + +def _item_shorthand(cls, type, name, expr="", description=None, **kwargs): + """Helper to populate :attr:`MESSAGE.items` or :attr:`MACRO.items`.""" + assert name not in cls.items + cls.items[name] = Item(name, type, expr, description=description, **kwargs) + + +def _log_filter_pattern(cls: type[GAMSModel]) -> "re.Pattern": + """Return a compiled :class:`re.Pattern` for filtering log messages. + + Messages like the following are matched: + + - "Existing index sets of 'FOO' [] do not match …" + - "Existing index names of 'BAR' [] do not match …" + + …where "FOO" or "BAR" are :any:`ItemType.EQU` or :any:`ItemType.VAR` among + the :attr:`cls.items `. Such log entries are generated by + :meth:`ixmp. + + The result is :func:`functools.cache`'d, thus only generated once. + """ + # Names of EQU or VAR-type items + names = sorted(k for k, v in cls.items.items() if v.type & ItemType.SOLUTION) + + # Expression matching log message from ixmp.model.base.Model.initialize_items() + return re.compile( + rf"Existing index (set|name)s of '({'|'.join(names)})' \[\] do not match" + ) diff --git a/message_ix/core.py b/message_ix/core.py old mode 100755 new mode 100644 index 290a5a939..007c1d097 --- a/message_ix/core.py +++ b/message_ix/core.py @@ -46,7 +46,7 @@ def __init__( if version == "new": scheme = scheme or "MESSAGE" - if scheme not in ("MESSAGE", None): + if scheme not in ("MESSAGE", "MESSAGE-MACRO", None): msg = f"Instantiate message_ix.Scenario with scheme {scheme}" raise ValueError(msg) @@ -61,7 +61,7 @@ def __init__( ) # Scheme returned by database - assert self.scheme == "MESSAGE", self.scheme + assert self.scheme in ("MESSAGE", "MESSAGE-MACRO"), self.scheme # Utility methods used by .equ(), .par(), .set(), and .var() @@ -793,8 +793,8 @@ def add_macro( -------- :ref:`macro-input-data` """ - from .macro import EXPERIMENTAL, add_model_data, calibrate - from .models import MACRO + from .macro import MACRO + from .macro.calibrate import EXPERIMENTAL, add_model_data, calibrate # Display a warning log.warning(EXPERIMENTAL) diff --git a/message_ix/macro/__init__.py b/message_ix/macro/__init__.py new file mode 100644 index 000000000..878519bd2 --- /dev/null +++ b/message_ix/macro/__init__.py @@ -0,0 +1,77 @@ +from collections.abc import MutableMapping +from functools import partial + +from ixmp.backend import ItemType + +from message_ix.common import GAMSModel, Item, _boollike, _item_shorthand + + +class MACRO(GAMSModel): + """Model class for MACRO.""" + + name = "MACRO" + + #: All equations, parameters, sets, and variables in the MACRO formulation. + items: MutableMapping[str, Item] = dict() + + #: MACRO uses the GAMS ``break;`` statement, and thus requires GAMS 24.8.1 or later. + GAMS_min_version = "24.8.1" + + keyword_to_solve_arg = [("concurrent", _boollike, "MACRO_CONCURRENT")] + + @classmethod + def initialize(cls, scenario, with_data=False): + """Initialize the model structure.""" + # NB some scenarios already have these items. This method simply adds any + # missing items. + + # Initialize the ixmp items for MACRO + cls.initialize_items(scenario, {k: v.to_dict() for k, v in cls.items.items()}) + + +equ = partial(_item_shorthand, MACRO, ItemType.EQU) +par = partial(_item_shorthand, MACRO, ItemType.PAR) +_set = partial(_item_shorthand, MACRO, ItemType.SET) +var = partial(_item_shorthand, MACRO, ItemType.VAR) + + +#: ixmp items (sets, parameters, variables, and equations) for MACRO. +_set("sector") +_set("mapping_macro_sector", "sector c l") +par("MERtoPPP", "n y") +par("aeei", "n sector y") +par("cost_MESSAGE", "n y") +par("demand_MESSAGE", "n sector y") +par("depr", "node") +par("drate", "node") +par("esub", "node") +par("gdp_calibrate", "n y") +par("grow", "n y") +par("historical_gdp", "n y") +par("kgdp", "node") +par("kpvs", "node") +par("lakl", "node") +par("lotol", "node") +par("prfconst", "n sector") +par("price_MESSAGE", "n sector y") +var("C", "n y", "Total consumption") +var("COST_NODAL", "n y") +var("COST_NODAL_NET", "n y", "Net of trade and emissions cost") +var("DEMAND", "n c l y h") +var("EC", "n y") +var("GDP", "n y") +var("I", "n y", "Total investment") +var("K", "n y") +var("KN", "n y") +var("MAX_ITER", "") +var("N_ITER", "") +var("NEWENE", "n sector y") +var("PHYSENE", "n sector y") +var("PRICE", "n c l y h") +var("PRODENE", "n sector y") +var("UTILITY", "") +var("Y", "n y") +var("YN", "n y") +var("aeei_calibrate", "n sector y") +var("grow_calibrate", "n y") +equ("COST_ACCOUNTING_NODAL", "n y") diff --git a/message_ix/macro.py b/message_ix/macro/calibrate.py similarity index 99% rename from message_ix/macro.py rename to message_ix/macro/calibrate.py index 8dfc09b48..7b0a27ce9 100644 --- a/message_ix/macro.py +++ b/message_ix/macro/calibrate.py @@ -483,7 +483,7 @@ def _validate_data(name: Optional[str], df: "DataFrame", s: Structures) -> list: list of str Dimensions/index sets of the validated MESSAGEix parameter. """ - from .models import MACRO + from . import MACRO # Check required dimensions if name is None: @@ -670,7 +670,7 @@ def prepare_computer( """ from ixmp.backend import ItemType - from .models import MACRO + from . import MACRO if not base.has_solution(): raise RuntimeError("Scenario must have a solution to add MACRO") diff --git a/message_ix/message.py b/message_ix/message.py new file mode 100644 index 000000000..135e7ff05 --- /dev/null +++ b/message_ix/message.py @@ -0,0 +1,867 @@ +import logging +from collections.abc import MutableMapping +from functools import partial +from warnings import warn + +import ixmp.model.gams +import pandas as pd +from ixmp.backend import ItemType +from ixmp.backend.jdbc import JDBCBackend +from ixmp.util import maybe_check_out, maybe_commit +from ixmp.util.ixmp4 import is_ixmp4backend + +from .common import ( + GAMSModel, + Item, + _boollike, + _filter_log_initialize_items, + _item_shorthand, +) + +log = logging.getLogger(__name__) + + +def _check_structure(scenario: "ixmp.Scenario"): + """Check dimensionality of some items related to the storage representation. + + Yields a sequence of 4-tuples: + + 1. Item name. + 2. Item ix_type. + 3. Number of data points in the item; -1 if it does not exist in `scenario`. + 4. A warning/error message, *if* the index names/sets do not match those in + :attr:`.MESSAGE.items` and the item contains data. Otherwise, the message is an + empty string. + """ + if scenario.has_solution(): + return + + # NB could rename this e.g. _check_structure_0 if there are multiple such methods + for name in ("storage_initial", "storage_self_discharge", "map_tec_storage"): + info = MESSAGE.items[name] + message = "" + N = -1 # Item does not exist; default + + try: + # Retrieve the index names and data length of the item + idx_names = tuple(scenario.idx_names(name)) + N = len(getattr(scenario, info.ix_type)(name)) + except KeyError: + pass + else: + # Item exists + expected_names = info.dims or info.coords + if expected_names != idx_names and N > 0: + message = ( + f"{info.ix_type} {name!r} has data with dimensions {idx_names!r}" + f" != {expected_names!r} and cannot be solved; try expand_dims()" + ) + finally: + yield name, info.ix_type, N, message + + +class MESSAGE(GAMSModel): + """Model class for MESSAGE.""" + + name = "MESSAGE" + + #: All equations, parameters, sets, and variables in the MESSAGE formulation. + items: MutableMapping[str, Item] = dict() + + keyword_to_solve_arg = [("cap_comm", _boollike, "MESSAGE_CAP_COMM")] + + @staticmethod + def enforce(scenario: "ixmp.Scenario") -> None: + """Enforce data consistency in `scenario`.""" + # Raise an exception if any of the storage items have incorrect dimensions, i.e. + # non-empty error messages + messages: list[str] = list( + filter(None, [msg for *_, msg in _check_structure(scenario)]) + ) + if messages: + raise ValueError("\n".join(messages)) + + # Check masks ("mapping sets") that indicate which elements of corresponding + # parameters are active/non-zero. Note that there are other masks currently + # handled in JDBCBackend. For the moment, this code does not backstop that + # behaviour. + # TODO Extend to handle all masks, e.g. for new backends. + for par_name in ("capacity_factor",): + # Name of the corresponding set + set_name = f"is_{par_name}" + + # Existing and expected contents + existing = scenario.set(set_name) + par_data = scenario.par(par_name) + assert isinstance(par_data, pd.DataFrame) and isinstance( + existing, pd.DataFrame + ) + expected = par_data.drop(columns=["value", "unit"]) + + if existing.equals(expected): + continue # Contents are as expected; do nothing + + # Not consistent; empty and then re-populate the set + with scenario.transact(f"Enforce consistency of {set_name} and {par_name}"): + scenario.remove_set(set_name, existing) + scenario.add_set(set_name, expected) + + @classmethod + def initialize(cls, scenario: "ixmp.Scenario") -> None: + """Set up `scenario` with required sets and parameters for MESSAGE. + + See Also + -------- + :attr:`items` + """ + from message_ix.core import Scenario + from message_ix.util.ixmp4 import platform_compat + from message_ix.util.scenario_setup import add_default_data + + # Adjust the Platform on which `scenario` is stored for compatibility between + # ixmp.{IXMP4,JDBC}Backend. Because message_ix does not subclass Platform, this + # is the earliest opportunity to make these adjustments. + platform_compat(scenario.platform) + + # Check for storage items that may contain incompatible data or need to be + # re-initialized + state = None + for name, ix_type, N, message in _check_structure(scenario): + if len(message): + warn(message) # Existing, incompatible data → conspicuous warning + elif N == 0: + # Existing, empty item → remove, even if it has the correct dimensions. + state = maybe_check_out(scenario, state) + getattr(scenario, f"remove_{ix_type}")(name) + + # Collect items to initialize + items = {k: v.to_dict() for k, v in cls.items.items()} + + # Prior to message_ix v1.2.0, COMMODITY_BALANCE was the name of an equation in + # the GAMS source (see .tests.test_legacy_version for an example). From v1.2.0 + # to v3.11.0, it was a GAMS macro, and thus not stored using ixmp. From v3.12.0 + # it is a variable. Do not try to initialize the variable if the equation is + # present. + if scenario.has_equ("COMMODITY_BALANCE"): + items.pop("COMMODITY_BALANCE") + + # Remove balance_equality for JDBC, where it seems to cause errors + if isinstance(scenario.platform._backend, JDBCBackend): + items.pop("balance_equality") + + # Hide verbose log messages if `scenario` was created with message_ix <3.10 and + # is being loaded with v3.11 or later + with _filter_log_initialize_items(cls): + # Initialize the ixmp items for MESSAGE + cls.initialize_items(scenario, items) + + if not isinstance(scenario, Scenario): + # Narrow type of `scenario` + # NB This should only occur if code constructs ixmp.Scenario(…, + # scheme="MESSAGE"), instead of message_ix.Scenario directly. User code + # *should* never do this, but it occurs in .test_models.test_initialize() + return + + # NOTE I tried transcribing this from ixmp_source as-is, but the MESSAGE class + # defined in models.py takes care of setting up the Scenario -- except for + # adding default data. + # ixmp_source does other things, too, which I don't think we need here, but I've + # kept them in for completeness for now. + + # ixmp_source first sets up a Scenario and adds default data + # models.MESSAGE seems to do the setup for us in all cases, while + # add_default_data() only adds missing items, so can always run. + # TODO Is this correct? + # if version == "new": + # # If the Scenario already exists, we don't need these two + # set_up_scenario(s=self) + add_default_data(scenario=scenario) + + # TODO We don't seem to need this, but if we do, give them better names + # self.tecParList = [ + # parameter_info for parameter_info in PARAMETERS if parameter_info.is_tec + # ] + # self.tecActParList = [ + # parameter_info + # for parameter_info in PARAMETERS + # if parameter_info.is_tec_act + # ] + + # TODO the following could be activated in ixmp_source through the flag + # parameter `sanity_checks`. This 'sanity_check' (there are more, s.b.) is + # generally only active when loading a scenario from the DB (unless explicitly + # loading via ID, in which case it's also inactive). We don't distinguish + # loading from the DB and some tutorials failed, so disable. + # ensure_required_indexsets_have_data(s=self) + + # TODO It does not seem useful to construct these because some required + # indexsets won't have any data in them yet. They do get run in imxp_source at + # this point, though. + # compose_maps(scenario=scenario) + + # Commit if anything was removed + maybe_commit(scenario, bool(state), f"{cls.__name__}.initialize") + + def run(self, scenario: "ixmp.Scenario") -> None: + from message_ix.core import Scenario + from message_ix.util.gams_io import ( + add_auxiliary_items_to_container_data_list, + add_default_data_to_container_data_list, + store_message_version, + ) + from message_ix.util.scenario_data import REQUIRED_EQUATIONS, REQUIRED_VARIABLES + from message_ix.util.scenario_setup import ( + compose_maps, + ensure_required_indexsets_have_data, + ) + + assert isinstance(scenario, Scenario) # Narrow type + + # Run the sanity checks + ensure_required_indexsets_have_data(scenario=scenario) + + compose_maps(scenario=scenario) + + if is_ixmp4backend(scenario.platform._backend): + # ixmp.model.gams.GAMSModel.__init__() creates the container_data attribute + # from its .defaults and any user kwargs + + # Add `MESSAGE_ix_version` parameter for validation by GAMS + store_message_version(container_data=self.container_data) + + # TODO Why is this a dedicated function? + # Add default data for some `Table`s to container data + for name in ("cat_tec", "type_tec_land"): + add_default_data_to_container_data_list( + container_data=self.container_data, name=name, scenario=scenario + ) + + # Add automatically created helper items to container data + add_auxiliary_items_to_container_data_list( + container_data=self.container_data, scenario=scenario + ) + + # Request only required Equations per default + self.equ_list = self.equ_list or [] + self.equ_list.extend(equation.gams_name for equation in REQUIRED_EQUATIONS) + self.equ_list.append("OBJECTIVE") + + # Request only required Variables per default + self.var_list = self.var_list or [] + self.var_list.extend(variable.gams_name for variable in REQUIRED_VARIABLES) + + super().run(scenario) + + +equ = partial(_item_shorthand, MESSAGE, ItemType.EQU) +par = partial(_item_shorthand, MESSAGE, ItemType.PAR) +_set = partial(_item_shorthand, MESSAGE, ItemType.SET) +var = partial(_item_shorthand, MESSAGE, ItemType.VAR) + + +# Index sets +_set("commodity") +_set("emission") +_set("grade") +_set("land_scenario") +_set("land_type") +_set("level_storage", description="Storage level") +_set("level") +_set("lvl_spatial") +_set("lvl_temporal") +_set("mode") +_set("node") +_set("rating") +_set("relation") +_set("shares") +_set("storage_tec", description="Storage reservoir technology") +_set("technology") +_set("time") +_set("time_relative") +_set("type_addon") +_set("type_emission") +_set("type_node") +_set("type_relation") +_set("type_tec") +_set("type_year") +_set("year") + +# Indexed sets +_set("addon", "t") +_set("balance_equality", "c l") +_set("cat_addon", "type_addon ta") +_set("cat_emission", "type_emission e") +_set("cat_node", "type_node n") +_set("cat_relation", "type_relation r") +_set("cat_tec", "type_tec t") +_set("cat_year", "type_year y") +_set("is_capacity_factor", "nl t yv ya h") +_set("level_renewable", "l") +_set("level_resource", "l") +_set("level_stocks", "l") +_set("map_node", "node_parent n") +_set("map_shares_commodity_share", "shares ns n type_tec m c l") +_set("map_shares_commodity_total", "shares ns n type_tec m c l") +_set("map_spatial_hierarchy", "lvl_spatial n node_parent") +_set("map_tec_addon", "t type_addon") +_set( + "map_tec_storage", + "n t m ts ms l c lvl_temporal", + description="Mapping of storage reservoir to charger/discharger", +) +_set("map_temporal_hierarchy", "lvl_temporal h time_parent") +_set("map_time", "time_parent h") +_set("type_tec_land", "type_tec") + +# Parameters +par("abs_cost_activity_soft_lo", "nl t ya h") +par("abs_cost_activity_soft_up", "nl t ya h") +par("abs_cost_new_capacity_soft_lo", "nl t yv") +par("abs_cost_new_capacity_soft_up", "nl t yv") +par("addon_conversion", "n t yv ya m h type_addon") +par("addon_lo", "n t ya m h type_addon") +par("addon_up", "n t ya m h type_addon") +par("bound_activity_lo", "nl t ya m h") +par("bound_activity_up", "nl t ya m h") +par("bound_emission", "n type_emission type_tec type_year") +par("bound_extraction_up", "n c g y") +par("bound_new_capacity_lo", "nl t yv") +par("bound_new_capacity_up", "nl t yv") +par("bound_total_capacity_lo", "nl t ya") +par("bound_total_capacity_up", "nl t ya") +par("capacity_factor", "nl t yv ya h") +par("commodity_stock", "n c l y") +par("construction_time", "nl t yv") +par("demand", "n c l y h") +par("duration_period", "y") +par("duration_time", "h") +par("dynamic_land_lo", "n s y u") +par("dynamic_land_up", "n s y u") +par("emission_factor", "nl t yv ya m e") +par("emission_scaling", "type_emission e") +par("fix_cost", "nl t yv ya") +par("fixed_activity", "nl t yv ya m h") +par("fixed_capacity", "nl t yv ya") +par("fixed_extraction", "n c g y") +par("fixed_land", "n s y") +par("fixed_new_capacity", "nl t yv") +par("fixed_stock", "n c l y") +par("flexibility_factor", "nl t yv ya m c l h q") +par("growth_activity_lo", "nl t ya h") +par("growth_activity_up", "nl t ya h") +par("growth_land_lo", "n y u") +par("growth_land_scen_lo", "n s y") +par("growth_land_scen_up", "n s y") +par("growth_land_up", "n y u") +par("growth_new_capacity_lo", "nl t yv") +par("growth_new_capacity_up", "nl t yv") +par("historical_activity", "nl t ya m h") +par("historical_emission", "n type_emission type_tec type_year") +par("historical_extraction", "n c g y") +par("historical_gdp", "n y") +par("historical_land", "n s y") +par("historical_new_capacity", "nl t yv") +par("initial_activity_lo", "nl t ya h") +par("initial_activity_up", "nl t ya h") +par("initial_land_lo", "n y u") +par("initial_land_scen_lo", "n s y") +par("initial_land_scen_up", "n s y") +par("initial_land_up", "n y u") +par("initial_new_capacity_lo", "nl t yv") +par("initial_new_capacity_up", "nl t yv") +par("input", "nl t yv ya m no c l h ho") +par("input_cap_new", "nl t yv no c l h") +par("input_cap_ret", "nl t yv no c l h") +par("input_cap", "nl t yv ya no c l h") +par("interestrate", "year") +par("inv_cost", "nl t yv") +par("land_cost", "n s y") +par("land_emission", "n s y e") +par("land_input", "n s y c l h") +par("land_output", "n s y c l h") +par("land_use", "n s y u") +par("level_cost_activity_soft_lo", "nl t ya h") +par("level_cost_activity_soft_up", "nl t ya h") +par("level_cost_new_capacity_soft_lo", "nl t yv") +par("level_cost_new_capacity_soft_up", "nl t yv") +par("min_utilization_factor", "nl t yv ya") +par("operation_factor", "nl t yv ya") +par("output_cap_new", "nl t yv nd c l h") +par("output_cap_ret", "nl t yv nd c l h") +par("output_cap", "nl t yv ya nd c l h") +par("output", "nl t yv ya m nd c l h hd") +par("peak_load_factor", "n c l y h") +par("rating_bin", "n t ya c l h q") +par("ref_activity", "nl t ya m h") +par("ref_extraction", "n c g y") +par("ref_new_capacity", "nl t yv") +par("ref_relation", "r nr yr") +par("relation_activity", "r nr yr nl t ya m") +par("relation_cost", "r nr yr") +par("relation_lower", "r nr yr") +par("relation_new_capacity", "r nr yr t") +par("relation_total_capacity", "r nr yr t") +par("relation_upper", "r nr yr") +par("reliability_factor", "n t ya c l h q") +par("renewable_capacity_factor", "n c g l y") +par("renewable_potential", "n c g l y") +par("resource_cost", "n c g y") +par("resource_remaining", "n c g y") +par("resource_volume", "n c g") +par("share_commodity_lo", "shares ns ya h") +par("share_commodity_up", "shares ns ya h") +par("share_mode_lo", "shares ns t m ya h") +par("share_mode_up", "shares ns t m ya h") +par("soft_activity_lo", "nl t ya h") +par("soft_activity_up", "nl t ya h") +par("soft_new_capacity_lo", "nl t yv") +par("soft_new_capacity_up", "nl t yv") +par("storage_initial", "n t m l c y h", "Initial amount of storage") +par( + "storage_self_discharge", + "n t m l c y h", + "Storage losses as a percentage of installed capacity", +) +par("subsidy", "nl type_tec ya") +par("tax_emission", "node type_emission type_tec type_year") +par("tax", "nl type_tec ya") +par("technical_lifetime", "nl t yv") +par("time_order", "lvl_temporal h", "Order of sub-annual time slices") +par("var_cost", "nl t yv ya m h") + +# Variables +var( + "ACT_LO", + "n t y h", + "Relaxation variable for dynamic constraints on activity (downwards)", +) +var( + "ACT_RATING", + "n t yv ya c l h q", + "Auxiliary variable for distributing total activity of a technology to a number of" + " 'rating bins'", +) +var( + "ACT_UP", + "n t y h", + "Relaxation variable for dynamic constraints on activity (upwards)", +) +var("ACT", "nl t yv ya m h", "Activity of technology") +var("CAP_FIRM", "n t c l y", "Capacity counting towards system reliability constraints") +var( + "CAP_NEW_LO", + "n t y", + "Relaxation variable for dynamic constraints on new capacity (downwards)", +) +var( + "CAP_NEW_UP", + "n t y", + "Relaxation variable for dynamic constraints on new capacity (upwards)", +) +var("CAP_NEW", "nl t yv", "New capacity") +var("CAP", "nl t yv ya", "Total installed capacity") +var("COMMODITY_BALANCE", "n c l y h", "Balance of commodity flow") +var( + "COMMODITY_USE", + "n c l y", + "Total amount of a commodity & level that was used or consumed", +) +var( + "COST_NODAL_NET", + "n y", + "System costs at the node level over time including effects of energy trade", +) +var("COST_NODAL", "n y", "System costs at the node level over time") +var("DEMAND", "n c l y h", "Demand") +var("EMISS", "n e type_tec y", "Aggregate emissions by technology type") +var("EXT", "n c g y", "Extraction of fossil resources") +var( + "GDP", + "n y", + "Gross domestic product (GDP) in market exchange rates for MACRO reporting", +) +var("LAND", "n s y", "Share of given land-use scenario") +var("OBJ", "", "Objective value of the optimisation problem (scalar)") +var( + "PRICE_COMMODITY", + "n c l y h", + "Commodity price (derived from marginals of COMMODITY_BALANCE constraint)", +) +var( + "PRICE_EMISSION", + "n type_emission type_tec y", + "Emission price (derived from marginals of EMISSION_EQUIVALENCE constraint)", +) +var( + "REL", + "r nr yr", + "Auxiliary variable for left-hand side of user-defined relations", +) +var( + "REN", + "n t c g y h", + "Activity of renewables specified per renewables grade", +) +var("SLACK_ACT_BOUND_LO", "n t y m h", "Slack variable for lower bound on activity") +var("SLACK_ACT_BOUND_UP", "n t y m h", "Slack variable for upper bound on activity") +var( + "SLACK_ACT_DYNAMIC_LO", + "n t y h", + "Slack variable for dynamic activity constraint relaxation (downwards)", +) +var( + "SLACK_ACT_DYNAMIC_UP", + "n t y h", + "Slack variable for dynamic activity constraint relaxation (upwards)", +) +var( + "SLACK_CAP_NEW_BOUND_LO", + "n t y", + "Slack variable for bound on new capacity (downwards)", +) +var( + "SLACK_CAP_NEW_BOUND_UP", + "n t y", + "Slack variable for bound on new capacity (upwards)", +) +var( + "SLACK_CAP_NEW_DYNAMIC_LO", + "n t y", + "Slack variable for dynamic new capacity constraint (downwards)", +) +var( + "SLACK_CAP_NEW_DYNAMIC_UP", + "n t y", + "Slack variable for dynamic new capacity constraint (upwards)", +) +var( + "SLACK_CAP_TOTAL_BOUND_LO", + "n t y", + "Slack variable for lower bound on total installed capacity", +) +var( + "SLACK_CAP_TOTAL_BOUND_UP", + "n t y", + "Slack variable for upper bound on total installed capacity", +) +var( + "SLACK_COMMODITY_EQUIVALENCE_LO", + "n c l y h", + "Slack variable for commodity balance (downwards)", +) +var( + "SLACK_COMMODITY_EQUIVALENCE_UP", + "n c l y h", + "Slack variable for commodity balance (upwards)", +) +var( + "SLACK_LAND_SCEN_LO", + "n s y", + "Slack variable for dynamic land scenario constraint relaxation (downwards)", +) +var( + "SLACK_LAND_SCEN_UP", + "n s y", + "Slack variable for dynamic land scenario constraint relaxation (upwards)", +) +var( + "SLACK_LAND_TYPE_LO", + "n y u", + "Slack variable for dynamic land type constraint relaxation (downwards)", +) +var( + "SLACK_LAND_TYPE_UP", + "n y u", + "Slack variable for dynamic land type constraint relaxation (upwards)", +) +var( + "SLACK_RELATION_BOUND_LO", + "r n y", + "Slack variable for lower bound of generic relation", +) +var( + "SLACK_RELATION_BOUND_UP", + "r n y", + "Slack variable for upper bound of generic relation", +) +var("STOCK_CHG", "n c l y h", "Annual input into and output from stocks of commodities") +var("STOCK", "n c l y", "Total quantity in intertemporal stock (storage)") +var( + "STORAGE_CHARGE", + "n t m l c y h", + "Charging of storage in each time slice (negative for discharge)", +) +var( + "STORAGE", + "n t m l c y h", + "State of charge (SoC) of storage at each sub-annual time slice (positive)", +) + +# Equations +equ( + "ACTIVITY_BOUND_ALL_MODES_LO", + "n t y h", + "Lower bound on activity summed over all vintages and modes", +) +equ( + "ACTIVITY_BOUND_ALL_MODES_UP", + "n t y h", + "Upper bound on activity summed over all vintages and modes", +) +equ( + "ACTIVITY_BOUND_LO", "n t y m h", "Lower bound on activity summed over all vintages" +) +equ( + "ACTIVITY_BOUND_UP", "n t y m h", "Upper bound on activity summed over all vintages" +) +equ( + "ACTIVITY_BY_RATING", + "n t y c l h q", + "Constraint on auxiliary rating-specific activity variable by rating bin", +) +equ( + "ACTIVITY_CONSTRAINT_LO", + "n t y h", + "Dynamic constraint on the market penetration of a technology activity" + " (lower bound)", +) +equ( + "ACTIVITY_CONSTRAINT_UP", + "n t y h", + "Dynamic constraint on the market penetration of a technology activity" + " (upper bound)", +) +equ("ACTIVITY_RATING_TOTAL", "n t yv y c l h", "Equivalence of `ACT_RATING` to `ACT`") +equ( + "ACTIVITY_SOFT_CONSTRAINT_LO", + "n t y h", + "Bound on relaxation of the dynamic constraint on market penetration (lower bound)", +) +equ( + "ACTIVITY_SOFT_CONSTRAINT_UP", + "n t y h", + "Bound on relaxation of the dynamic constraint on market penetration (upper bound)", +) +equ( + "ADDON_ACTIVITY_LO", + "n type_addon y m h", + "Addon technology activity lower constraint", +) +equ( + "ADDON_ACTIVITY_UP", + "n type_addon y m h", + "Addon-technology activity upper constraint", +) +# TODO I think inv_tec is defined only when writing out to GAMS, while this equation +# will need it to exist first -- but not populated, so just create it as a required set +# without data? +equ( + "CAPACITY_CONSTRAINT", + "n inv_tec yv y h", + "Capacity constraint for technology (by sub-annual time slice)", +) +equ( + "CAPACITY_MAINTENANCE_HIST", + "n inv_tec yv first_period", + "Constraint for capacity maintenance historical installation (built before " + "start of model horizon)", +) +equ( + "CAPACITY_MAINTENANCE_NEW", + "n inv_tec yv ya", + "Constraint for capacity maintenance of new capacity built in the current " + "period (vintage == year)", +) +equ( + "CAPACITY_MAINTENANCE", + "n inv_tec yv y", + "Constraint for capacity maintenance over the technical lifetime", +) +equ( + "COMMODITY_BALANCE_GT", + "n c l y h", + "Commodity supply greater than or equal demand", +) +equ("COMMODITY_BALANCE_LT", "n c l y h", "Commodity supply lower than or equal demand") +equ( + "COMMODITY_USE_LEVEL", + "n c l y h", + "Aggregate use of commodity by level as defined by total input into technologies", +) +equ("COST_ACCOUNTING_NODAL", "n y", "Cost accounting aggregated to the node") +equ( + "DYNAMIC_LAND_SCEN_CONSTRAINT_LO", + "n s y", + "Dynamic constraint on land scenario change (lower bound)", +) +equ( + "DYNAMIC_LAND_SCEN_CONSTRAINT_UP", + "n s y", + "Dynamic constraint on land scenario change (upper bound)", +) +equ( + "DYNAMIC_LAND_TYPE_CONSTRAINT_LO", + "n y u", + "Dynamic constraint on land-use change (lower bound)", +) +equ( + "DYNAMIC_LAND_TYPE_CONSTRAINT_UP", + "n y u", + "Dynamic constraint on land-use change (upper bound)", +) +equ( + "EMISSION_CONSTRAINT", + "n type_emission type_tec type_year", + "Nodal-regional-global constraints on emissions (by category)", +) +equ( + "EMISSION_EQUIVALENCE", + "n e type_tec y", + "Auxiliary equation to simplify the notation of emissions", +) +equ("EXTRACTION_BOUND_UP", "n c g y", "Upper bound on extraction (by grade)") +equ( + "EXTRACTION_EQUIVALENCE", + "n c y", + "Auxiliary equation to simplify the resource extraction formulation", +) +equ( + "FIRM_CAPACITY_PROVISION", + "n inv_tec y c l h", + "Contribution of dispatchable technologies to auxiliary firm-capacity variable", +) +equ( + "LAND_CONSTRAINT", + "n y", + "Constraint on total land use (partial sum of `LAND` on `land_scenario` is 1)", +) +equ( + "MIN_UTILIZATION_CONSTRAINT", + "n inv_tec yv y", + "Constraint for minimum yearly operation (aggregated over the course of a year)", +) +equ( + "NEW_CAPACITY_BOUND_LO", + "n inv_tec y", + "Lower bound on technology capacity investment", +) +equ( + "NEW_CAPACITY_BOUND_UP", + "n inv_tec y", + "Upper bound on technology capacity investment", +) +equ( + "NEW_CAPACITY_CONSTRAINT_LO", + "n inv_tec y", + "Dynamic constraint on capacity investment (lower bound)", +) +equ( + "NEW_CAPACITY_CONSTRAINT_UP", + "n inv_tec y", + "Dynamic constraint for capacity investment (learning and spillovers upper bound)", +) +equ( + "NEW_CAPACITY_SOFT_CONSTRAINT_LO", + "n inv_tec y", + "Bound on soft relaxation of dynamic new capacity constraints (downwards)", +) +equ( + "NEW_CAPACITY_SOFT_CONSTRAINT_UP", + "n inv_tec y", + "Bound on soft relaxation of dynamic new capacity constraints (upwards)", +) +equ("OBJECTIVE", "", "Objective value of the optimisation problem") +equ( + "OPERATION_CONSTRAINT", + "n inv_tec yv y", + "Constraint on maximum yearly operation (scheduled down-time for maintenance)", +) +equ("RELATION_CONSTRAINT_LO", "r n y", "Lower bound of relations (linear constraints)") +equ("RELATION_CONSTRAINT_UP", "r n y", "Upper bound of relations (linear constraints)") +equ( + "RELATION_EQUIVALENCE", + "r n y", + "Auxiliary equation to simplify the implementation of relations", +) +equ( + "RENEWABLES_CAPACITY_REQUIREMENT", + "n inv_tec c y", + "Lower bound on required overcapacity when using lower grade potentials", +) +equ( + "RENEWABLES_EQUIVALENCE", + "n renewable_tec c y h", + "Equation to define the renewables extraction", +) +equ( + "RENEWABLES_POTENTIAL_CONSTRAINT", + "n c g y", + "Constraint on renewable resource potential", +) +equ( + "RESOURCE_CONSTRAINT", + "n c g y", + "Constraint on resources remaining in each period (maximum extraction per period)", +) +equ( + "RESOURCE_HORIZON", + "n c g", + "Constraint on extraction over entire model horizon (resource volume in place)", +) +equ( + "SHARE_CONSTRAINT_COMMODITY_LO", + "shares ns y h", + "Lower bounds on share constraints for commodities", +) +equ( + "SHARE_CONSTRAINT_COMMODITY_UP", + "shares ns y h", + "Upper bounds on share constraints for commodities", +) +equ( + "SHARE_CONSTRAINT_MODE_LO", + "shares n t m y h", + "Lower bounds on share constraints for modes of a given technology", +) +equ( + "SHARE_CONSTRAINT_MODE_UP", + "shares n t m y h", + "Upper bounds on share constraints for modes of a given technology", +) +equ("STOCKS_BALANCE", "n c l y", "Commodity inter-temporal balance of stocks") +# FIXME 'h2' or 'time2' are not indicative names, come up with better ones +equ( + "STORAGE_BALANCE_INIT", + "n ts m l c y h h2", + "Balance of the state of charge of storage at sub-annual time slices with " + "initial storage content", +) +# TODO Why is this using h2 and not h? +equ( + "STORAGE_BALANCE", + "n ts m l c y h2 lvl_temporal", + "Balance of the state of charge of storage", +) +equ( + "STORAGE_CHANGE", + "n ts m level_storage c y h", + "Change in the state of charge of storage", +) +equ( + "STORAGE_INPUT", + "n ts l c level_storage c2 m y h", + "Connecting an input commodity to maintain the activity of storage container " + "(not stored commodity)", +) +equ( + "SYSTEM_FLEXIBILITY_CONSTRAINT", + "n c l y h", + "Constraint on total system flexibility", +) +equ( + "SYSTEM_RELIABILITY_CONSTRAINT", + "n c l y h", + "Constraint on total system reliability (firm capacity)", +) +equ("TOTAL_CAPACITY_BOUND_LO", "n inv_tec y", "Lower bound on total installed capacity") +equ("TOTAL_CAPACITY_BOUND_UP", "n inv_tec y", "Upper bound on total installed capacity") diff --git a/message_ix/message_macro.py b/message_ix/message_macro.py new file mode 100644 index 000000000..7553637d0 --- /dev/null +++ b/message_ix/message_macro.py @@ -0,0 +1,27 @@ +from collections import ChainMap + +from .macro import MACRO +from .message import MESSAGE + + +class MESSAGE_MACRO(MESSAGE, MACRO): + """Model class for MESSAGE_MACRO.""" + + name = "MESSAGE-MACRO" + #: All equations, parameters, sets, and variables in the MESSAGE-MACRO formulation. + items = ChainMap(MESSAGE.items, MACRO.items) + + keyword_to_solve_arg = ( + MACRO.keyword_to_solve_arg + + MESSAGE.keyword_to_solve_arg + + [ + ("convergence_criterion", float, "CONVERGENCE_CRITERION"), + ("max_adjustment", float, "MAX_ADJUSTMENT"), + ("max_iteration", int, "MAX_ITERATION"), + ] + ) + + @classmethod + def initialize(cls, scenario, with_data=False): + MESSAGE.initialize(scenario) + MACRO.initialize(scenario, with_data) diff --git a/message_ix/model/MESSAGE/model_core.gms b/message_ix/model/MESSAGE/model_core.gms index 97122e544..18c263955 100644 --- a/message_ix/model/MESSAGE/model_core.gms +++ b/message_ix/model/MESSAGE/model_core.gms @@ -139,19 +139,27 @@ Variables * * Auxiliary variables * ^^^^^^^^^^^^^^^^^^^ -* =========================================================================== ======================================================================================================= -* Variable Explanatory text -* =========================================================================== ======================================================================================================= -* :math:`\text{DEMAND}_{n,c,l,y,h} \in \mathbb{R}` Demand level (in equilibrium with MACRO integration) -* :math:`\text{PRICE_COMMODITY}_{n,c,l,y,h} \in \mathbb{R}` Commodity price (undiscounted marginals of :ref:`commodity_balance_gt` and :ref:`commodity_balance_lt`) -* :math:`\text{PRICE_EMISSION}_{n,\widehat{e},\widehat{t},y} \in \mathbb{R}` Emission price (undiscounted marginals of :ref:`emission_equivalence`) -* :math:`\text{COST_NODAL_NET}_{n,y} \in \mathbb{R}` System costs at the node level net of energy trade revenues/cost -* :math:`\text{GDP}_{n,y} \in \mathbb{R}` Gross domestic product (GDP) in market exchange rates for MACRO reporting -* =========================================================================== ====================================================================================================== +* +* .. list-table:: +* :header-rows: 1 +* +* * - Variable +* - Explanatory text +* * - :math:`\text{DEMAND}_{n,c,l,y,h} \in \mathbb{R}` +* - Demand level (in equilibrium with MACRO integration) +* * - :math:`\text{PRICE_COMMODITY}_{n,c,l,y,h} \in \mathbb{R}` +* - Commodity price (undiscounted marginals of :ref:`commodity_balance_gt` and :ref:`commodity_balance_lt`) +* * - :math:`\text{PRICE_EMISSION}_{n,\widehat{e},\widehat{t},y} \in \mathbb{R}` +* - Emission price (undiscounted marginals of :ref:`emission_equivalence`) +* * - :math:`\text{COST_NODAL_NET}_{n,y} \in \mathbb{R}` +* - System costs at the node level net of energy trade revenues/cost +* * - :math:`\text{GDP}_{n,y} \in \mathbb{R}` +* - Gross domestic product (GDP) in market exchange rates for MACRO reporting * * .. warning:: -* Please be aware that transitioning from one period length to another for consecutive periods may result in false values of :math:`\text{PRICE_EMISSION}`. -* Please see `this issue `_ for further information. We are currently working on a fix. +* Please be aware that transitioning from one period length to another +* for consecutive periods may result in incorrect value for :math:`\text{PRICE_EMISSION}`. +* See :issue:`723` for further information and :pull:`726` for a potential resolution. *** Variables @@ -574,7 +582,7 @@ RESOURCE_HORIZON(node,commodity,grade)$( SUM(year$map_resource(node,commodity,gr * - Net |input| minus |output| of commodities based on technology activity (|ACT|). * - Net |land_input| minus |land_output| of commodities based on |LAND|. * - Inter-period transfers via |STOCK_CHG|. -* - If the :class:`MESSAGE` option :py:`cap_comm=True` is given, +* - If the :class:`.MESSAGE` option :py:`cap_comm=True` is given, * flows of commodities (e.g. ‘materials’) * associated with construction and retirement of technology capacity (|CAP|). * For |y0|, this representation requires |historical_new_capacity| parameter values, diff --git a/message_ix/model/README b/message_ix/model/README new file mode 100644 index 000000000..ac247f57f --- /dev/null +++ b/message_ix/model/README @@ -0,0 +1,2 @@ +This directory contains the GAMS implemenations of +MACRO, MESSAGE, and MESSAGE_MACRO. diff --git a/message_ix/models.py b/message_ix/models.py index 9d5d5f602..a8e7295ae 100644 --- a/message_ix/models.py +++ b/message_ix/models.py @@ -1,1271 +1,27 @@ -import logging -import re -from collections import ChainMap -from collections.abc import Iterator, Mapping, MutableMapping -from contextlib import contextmanager -from copy import copy -from dataclasses import InitVar, dataclass, field -from functools import cache, partial -from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from importlib import import_module from warnings import warn -import ixmp.model.gams -import pandas as pd -from ixmp import config -from ixmp.backend import ItemType -from ixmp.backend.jdbc import JDBCBackend -from ixmp.util import maybe_check_out, maybe_commit -from ixmp.util.ixmp4 import is_ixmp4backend - -if TYPE_CHECKING: - from logging import LogRecord - - from ixmp.types import InitializeItemsKwargs - - -log = logging.getLogger(__name__) - -#: Solver options used by :meth:`.Scenario.solve`. -DEFAULT_CPLEX_OPTIONS = { - "advind": 0, - "lpmethod": 4, - "threads": 4, - "epopt": 1e-6, +# Names formerly in this module, importable with deprecation warning. +_DEPRECATED_IMPORT = { + "DIMS": "common", + "Item": "common", + "MACRO": "macro", + "MESSAGE": "message", + "MESSAGE_MACRO": "message_macro", } -#: Common dimension name abbreviations mapped to tuples with: -#: -#: 1. the respective coordinate/index set, and -#: 2. the full dimension name. -DIMS = { - "c": ("commodity", "commodity"), - "c2": ("commodity", "commodity2"), - "e": ("emission", "emission"), - "first_period": ("year", "first_period"), - "g": ("grade", "grade"), - "h": ("time", "time"), - "h2": ("time", "time2"), - "hd": ("time", "time_dest"), - "ho": ("time", "time_origin"), - "inv_tec": ("technology", "inv_tec"), - "l": ("level", "level"), - "m": ("mode", "mode"), - "ms": ("mode", "storage_mode"), - "n": ("node", "node"), - "nd": ("node", "node_dest"), - "nl": ("node", "node_loc"), - "no": ("node", "node_origin"), - "node_parent": ("node", "node_parent"), - "nr": ("node", "node_rel"), - "ns": ("node", "node_share"), - "q": ("rating", "rating"), - "r": ("relation", "relation"), - "renewable_tec": ("technology", "renewable_tec"), - "s": ("land_scenario", "land_scenario"), - "t": ("technology", "technology"), - "ta": ("technology", "technology_addon"), - "time_parent": ("time", "time_parent"), - "tp": ("technology", "technology_primary"), - "ts": ("technology", "storage_tec"), - "u": ("land_type", "land_type"), - "y": ("year", "year"), - "ya": ("year", "year_act"), - "yr": ("year", "year_rel"), - "yv": ("year", "year_vtg"), -} - - -@dataclass -class Item: - """Description of an :mod:`ixmp` item: equation, parameter, set, or variable. - - Instances of this class carry only structural information, not data. - """ - - #: Name of the item. - name: str - - #: Type of the item, for instance :attr:`ItemType.PAR `. - type: "ixmp.backend.ItemType" - - #: String expression for :attr:`coords` and :attr:`dims`. Split on spaces and parsed - #: using :data:`DIMS` so that, for instance, "nl yv" results in entries for - #: for "node", "year" in :attr:`coords`, and "node_loc", "year_vtg" in :attr:`dims`. - expr: InitVar[str] = "" - - #: Coordinates of the item; that is, the names of sets that index its dimensions. - #: The same set name may be repeated if it indexes multiple dimensions. - coords: tuple[str, ...] = field(default_factory=tuple) - - #: Dimensions of the item. - dims: tuple[str, ...] = field(default_factory=tuple) - - #: Text description of the item. - description: Optional[str] = None - - def __post_init__(self, expr): - if expr == "": - return - - # Split on spaces. For each dimension, use an abbreviation (if one) exists, else - # the set name for both coords and dims - self.coords, self.dims = zip(*[DIMS.get(d, (d, d)) for d in expr.split()]) - - if self.dims == self.coords: - # No distinct dimension names; don't store these - self.dims = tuple() - - @property - def ix_type(self) -> str: - """Lower-case string form of :attr:`type`: "equ", "par", "set", or "var". - - Read-only. - """ - return str(self.type.name).lower() - - def to_dict(self) -> "InitializeItemsKwargs": - """Return the :class:`dict` representation used internally in :mod:`ixmp`.""" - result: "InitializeItemsKwargs" = dict( - ix_type=self.ix_type, idx_sets=self.coords - ) - if self.dims: - result["idx_names"] = self.dims - return result - - -def _item_shorthand(cls, type, name, expr="", description=None, **kwargs): - """Helper to populate :attr:`MESSAGE.items` or :attr:`MACRO.items`.""" - assert name not in cls.items - cls.items[name] = Item(name, type, expr, description=description, **kwargs) - - -def item(ix_type, expr, description: Optional[str] = None) -> "InitializeItemsKwargs": - """Return a dict with idx_sets and idx_names, given a string `expr`. - - .. deprecated:: 3.8.0 - - Instead, use :py:`Item(...).to_dict()` - """ - item = Item("", ItemType[ix_type.upper()], expr, description=description) - return item.to_dict() - - -def _template(*parts): - """Helper to make a template string relative to model_dir.""" - return str(Path("{model_dir}", *parts)) - - -class GAMSModel(ixmp.model.gams.GAMSModel): - """Extended :class:`ixmp.model.gams.GAMSModel` for MESSAGE & MACRO.""" - - #: Default model options. - defaults = ChainMap( - { - # New keys for MESSAGE & MACRO - "model_dir": Path(__file__).parent / "model", - # Override keys from GAMSModel - "model_file": _template("{model_name}_run.gms"), - "in_file": _template("data", "MsgData_{case}.gdx"), - "out_file": _template("output", "MsgOutput_{case}.gdx"), - "solve_args": [ - '--in="{in_file}"', - '--out="{out_file}"', - '--iter="{}"'.format( - _template("output", "MsgIterationReport_{case}.gdx") - ), - ], - # Disable the feature to put input/output GDX files, list files, etc. in a - # temporary directory - "use_temp_dir": False, - # Record versions of message_ix and ixmp in GDX I/O files - "record_version_packages": ("message_ix", "ixmp"), - }, - ixmp.model.gams.GAMSModel.defaults, - ) - - #: Mapping from model item (equation, parameter, set, or variable) names to - #: :class:`.Item` describing the item. - items: Mapping[str, Item] - - # Make default model options known to the class - model_dir: Path - - #: Optional minimum version of GAMS. - GAMS_min_version: Optional[str] = None - - #: Keyword arguments to map to GAMS `solve_args`. - keyword_to_solve_arg: list[tuple[str, type, str]] - - def __init__(self, name: Optional[str] = None, **model_options) -> None: - if gmv := self.GAMS_min_version: - # Check the minimum GAMS version. - version = ixmp.model.gams.gams_version() or "" - if version < gmv: - raise RuntimeError(f"{self.name} requires GAMS >= {gmv}; got {version}") - - # Convert optional `keyword` arguments to GAMS CLI arguments like ``--{target}`` - solve_args = [] - for keyword, callback, target in self.keyword_to_solve_arg: - try: - raw = model_options.pop(keyword) - solve_args.append(f"--{target}={callback(raw)!s}") - except KeyError: - pass - except ValueError: - raise ValueError(f"{keyword} = {raw}") - - # Update the default options with any user-provided options - model_options.setdefault("model_dir", config.get("message model dir")) - self.cplex_opts = copy(DEFAULT_CPLEX_OPTIONS) - self.cplex_opts.update(config.get("message solve options") or dict()) - self.cplex_opts.update(model_options.pop("solve_options", {})) - - super().__init__(name, **model_options) - - self.solve_args.extend(solve_args) - - def run(self, scenario: "ixmp.Scenario") -> None: - """Execute the model. - - GAMSModel creates a file named ``cplex.opt`` in the model directory containing - the “solve_options”, as described above. - - .. warning:: GAMSModel can solve Scenarios in two or more Python processes - simultaneously; but using *different* CPLEX options in each process may - produce unexpected results. - """ - # Ensure the data in `scenario` is consistent with the MESSAGE formulation - self.enforce(scenario) - - # If two runs are kicked off simultaneously with the same self.model_dir, then - # they will try to write the same optfile, and may write different contents. - # - # TODO Re-enable the 'use_temp_dir' feature from ixmp.GAMSModel (disabled above) - # so that cplex.opt will be specific to that directory. - - # Write CPLEX options into an options file - optfile = Path(self.model_dir).joinpath("cplex.opt") - lines = ("{} = {}".format(*kv) for kv in self.cplex_opts.items()) - optfile.write_text("\n".join(lines)) - log.info(f"Use CPLEX options {self.cplex_opts}") - - self.cplex_opts.update({"barcrossalg": 2}) - optfile2 = Path(self.model_dir).joinpath("cplex.op2") - lines2 = ("{} = {}".format(*kv) for kv in self.cplex_opts.items()) - optfile2.write_text("\n".join(lines2)) - - result = super().run(scenario) - - # In previous versions, the `cplex.opt` file(s) were removed at this point - # in the workflow. This has been removed due to issues when running - # scenarios asynchronously. - - return result - - -class _boollike(str): - """Handle a :class:`bool`-like argument; return :py:`"0"` or :py:`"1"`.""" - - def __new__(cls, value: Any): - if value in {"0", 0, False, "False"}: - return "0" - elif value in {"1", 1, True, "True"}: - return "1" - else: - raise ValueError - - -def _check_structure(scenario: "ixmp.Scenario"): - """Check dimensionality of some items related to the storage representation. - - Yields a sequence of 4-tuples: - - 1. Item name. - 2. Item ix_type. - 3. Number of data points in the item; -1 if it does not exist in `scenario`. - 4. A warning/error message, *if* the index names/sets do not match those in - :attr:`.MESSAGE.items` and the item contains data. Otherwise, the message is an - empty string. - """ - if scenario.has_solution(): - return - - # NB could rename this e.g. _check_structure_0 if there are multiple such methods - for name in ("storage_initial", "storage_self_discharge", "map_tec_storage"): - info = MESSAGE.items[name] - message = "" - N = -1 # Item does not exist; default - - try: - # Retrieve the index names and data length of the item - idx_names = tuple(scenario.idx_names(name)) - N = len(getattr(scenario, info.ix_type)(name)) - except KeyError: - pass - else: - # Item exists - expected_names = info.dims or info.coords - if expected_names != idx_names and N > 0: - message = ( - f"{info.ix_type} {name!r} has data with dimensions {idx_names!r}" - f" != {expected_names!r} and cannot be solved; try expand_dims()" - ) - finally: - yield name, info.ix_type, N, message - - -@cache -def _log_filter_pattern(cls: type[GAMSModel]) -> "re.Pattern": - """Return a compiled :class:`re.Pattern` for filtering log messages. - - Messages like the following are matched: - - - "Existing index sets of 'FOO' [] do not match …" - - "Existing index names of 'BAR' [] do not match …" - - …where "FOO" or "BAR" are :any:`ItemType.EQU` or :any:`ItemType.VAR` among - the :attr:`cls.items `. Such log entries are generated by - :meth:`ixmp. - - The result is :func:`functools.cache`'d, thus only generated once. - """ - # Names of EQU or VAR-type items - names = sorted(k for k, v in cls.items.items() if v.type & ItemType.SOLUTION) - - # Expression matching log message from ixmp.model.base.Model.initialize_items() - return re.compile( - rf"Existing index (set|name)s of '({'|'.join(names)})' \[\] do not match" - ) - - -@contextmanager -def _filter_log_initialize_items(cls: type[GAMSModel]) -> Iterator[None]: - """Context manager to filter log messages related to existing items.""" - pattern = _log_filter_pattern(cls) - def _filter(record: "LogRecord") -> bool: - # Retrieve the compiled expression - return not pattern.match(record.msg) - - # Attach the filter to the ixmp.model logger - logger = logging.getLogger("ixmp.model.base") - logger.addFilter(_filter) +def __getattr__(name: str): try: - yield - finally: - logger.removeFilter(_filter) - - -class MESSAGE(GAMSModel): - """Model class for MESSAGE.""" - - name = "MESSAGE" - - #: All equations, parameters, sets, and variables in the MESSAGE formulation. - items: MutableMapping[str, Item] = dict() - - keyword_to_solve_arg = [("cap_comm", _boollike, "MESSAGE_CAP_COMM")] - - @staticmethod - def enforce(scenario: "ixmp.Scenario") -> None: - """Enforce data consistency in `scenario`.""" - # Raise an exception if any of the storage items have incorrect dimensions, i.e. - # non-empty error messages - messages: list[str] = list( - filter(None, [msg for *_, msg in _check_structure(scenario)]) - ) - if messages: - raise ValueError("\n".join(messages)) - - # Check masks ("mapping sets") that indicate which elements of corresponding - # parameters are active/non-zero. Note that there are other masks currently - # handled in JDBCBackend. For the moment, this code does not backstop that - # behaviour. - # TODO Extend to handle all masks, e.g. for new backends. - for par_name in ("capacity_factor",): - # Name of the corresponding set - set_name = f"is_{par_name}" - - # Existing and expected contents - existing = scenario.set(set_name) - par_data = scenario.par(par_name) - assert isinstance(par_data, pd.DataFrame) and isinstance( - existing, pd.DataFrame - ) - expected = par_data.drop(columns=["value", "unit"]) - - if existing.equals(expected): - continue # Contents are as expected; do nothing - - # Not consistent; empty and then re-populate the set - with scenario.transact(f"Enforce consistency of {set_name} and {par_name}"): - scenario.remove_set(set_name, existing) - scenario.add_set(set_name, expected) - - @classmethod - def initialize(cls, scenario: "ixmp.Scenario") -> None: - """Set up `scenario` with required sets and parameters for MESSAGE. - - See Also - -------- - :attr:`items` - """ - from message_ix.core import Scenario - from message_ix.util.ixmp4 import platform_compat - from message_ix.util.scenario_setup import add_default_data - - # Adjust the Platform on which `scenario` is stored for compatibility between - # ixmp.{IXMP4,JDBC}Backend. Because message_ix does not subclass Platform, this - # is the earliest opportunity to make these adjustments. - platform_compat(scenario.platform) - - # Check for storage items that may contain incompatible data or need to be - # re-initialized - state = None - for name, ix_type, N, message in _check_structure(scenario): - if len(message): - warn(message) # Existing, incompatible data → conspicuous warning - elif N == 0: - # Existing, empty item → remove, even if it has the correct dimensions. - state = maybe_check_out(scenario, state) - getattr(scenario, f"remove_{ix_type}")(name) - - # Collect items to initialize - items = {k: v.to_dict() for k, v in cls.items.items()} - - # Prior to message_ix v1.2.0, COMMODITY_BALANCE was the name of an equation in - # the GAMS source (see .tests.test_legacy_version for an example). From v1.2.0 - # to v3.11.0, it was a GAMS macro, and thus not stored using ixmp. From v3.12.0 - # it is a variable. Do not try to initialize the variable if the equation is - # present. - if scenario.has_equ("COMMODITY_BALANCE"): - items.pop("COMMODITY_BALANCE") - - # Remove balance_equality for JDBC, where it seems to cause errors - if isinstance(scenario.platform._backend, JDBCBackend): - items.pop("balance_equality") - - # Hide verbose log messages if `scenario` was created with message_ix <3.10 and - # is being loaded with v3.11 or later - with _filter_log_initialize_items(cls): - # Initialize the ixmp items for MESSAGE - cls.initialize_items(scenario, items) - - if not isinstance(scenario, Scenario): - # Narrow type of `scenario` - # NB This should only occur if code constructs ixmp.Scenario(…, - # scheme="MESSAGE"), instead of message_ix.Scenario directly. User code - # *should* never do this, but it occurs in .test_models.test_initialize() - return - - # NOTE I tried transcribing this from ixmp_source as-is, but the MESSAGE class - # defined in models.py takes care of setting up the Scenario -- except for - # adding default data. - # ixmp_source does other things, too, which I don't think we need here, but I've - # kept them in for completeness for now. - - # ixmp_source first sets up a Scenario and adds default data - # models.MESSAGE seems to do the setup for us in all cases, while - # add_default_data() only adds missing items, so can always run. - # TODO Is this correct? - # if version == "new": - # # If the Scenario already exists, we don't need these two - # set_up_scenario(s=self) - add_default_data(scenario=scenario) - - # TODO We don't seem to need this, but if we do, give them better names - # self.tecParList = [ - # parameter_info for parameter_info in PARAMETERS if parameter_info.is_tec - # ] - # self.tecActParList = [ - # parameter_info - # for parameter_info in PARAMETERS - # if parameter_info.is_tec_act - # ] - - # TODO the following could be activated in ixmp_source through the flag - # parameter `sanity_checks`. This 'sanity_check' (there are more, s.b.) is - # generally only active when loading a scenario from the DB (unless explicitly - # loading via ID, in which case it's also inactive). We don't distinguish - # loading from the DB and some tutorials failed, so disable. - # ensure_required_indexsets_have_data(s=self) - - # TODO It does not seem useful to construct these because some required - # indexsets won't have any data in them yet. They do get run in imxp_source at - # this point, though. - # compose_maps(scenario=scenario) - - # Commit if anything was removed - maybe_commit(scenario, bool(state), f"{cls.__name__}.initialize") - - def run(self, scenario: "ixmp.Scenario") -> None: - from message_ix.core import Scenario - from message_ix.util.gams_io import ( - add_auxiliary_items_to_container_data_list, - add_default_data_to_container_data_list, - store_message_version, - ) - from message_ix.util.scenario_data import REQUIRED_EQUATIONS, REQUIRED_VARIABLES - from message_ix.util.scenario_setup import ( - compose_maps, - ensure_required_indexsets_have_data, - ) - - assert isinstance(scenario, Scenario) # Narrow type - - # Run the sanity checks - ensure_required_indexsets_have_data(scenario=scenario) - - compose_maps(scenario=scenario) - - if is_ixmp4backend(scenario.platform._backend): - # ixmp.model.gams.GAMSModel.__init__() creates the container_data attribute - # from its .defaults and any user kwargs - - # Add `MESSAGE_ix_version` parameter for validation by GAMS - store_message_version(container_data=self.container_data) - - # TODO Why is this a dedicated function? - # Add default data for some `Table`s to container data - for name in ("cat_tec", "type_tec_land"): - add_default_data_to_container_data_list( - container_data=self.container_data, name=name, scenario=scenario - ) - - # Add automatically created helper items to container data - add_auxiliary_items_to_container_data_list( - container_data=self.container_data, scenario=scenario - ) - - # Request only required Equations per default - self.equ_list = self.equ_list or [] - self.equ_list.extend(equation.gams_name for equation in REQUIRED_EQUATIONS) - self.equ_list.append("OBJECTIVE") - - # Request only required Variables per default - self.var_list = self.var_list or [] - self.var_list.extend(variable.gams_name for variable in REQUIRED_VARIABLES) - - super().run(scenario) - - -equ = partial(_item_shorthand, MESSAGE, ItemType.EQU) -par = partial(_item_shorthand, MESSAGE, ItemType.PAR) -_set = partial(_item_shorthand, MESSAGE, ItemType.SET) -var = partial(_item_shorthand, MESSAGE, ItemType.VAR) - - -# Index sets -_set("commodity") -_set("emission") -_set("grade") -_set("land_scenario") -_set("land_type") -_set("level_storage", description="Storage level") -_set("level") -_set("lvl_spatial") -_set("lvl_temporal") -_set("mode") -_set("node") -_set("rating") -_set("relation") -_set("shares") -_set("storage_tec", description="Storage reservoir technology") -_set("technology") -_set("time") -_set("time_relative") -_set("type_addon") -_set("type_emission") -_set("type_node") -_set("type_relation") -_set("type_tec") -_set("type_year") -_set("year") - -# Indexed sets -_set("addon", "t") -_set("balance_equality", "c l") -_set("cat_addon", "type_addon ta") -_set("cat_emission", "type_emission e") -_set("cat_node", "type_node n") -_set("cat_relation", "type_relation r") -_set("cat_tec", "type_tec t") -_set("cat_year", "type_year y") -_set("is_capacity_factor", "nl t yv ya h") -_set("level_renewable", "l") -_set("level_resource", "l") -_set("level_stocks", "l") -_set("map_node", "node_parent n") -_set("map_shares_commodity_share", "shares ns n type_tec m c l") -_set("map_shares_commodity_total", "shares ns n type_tec m c l") -_set("map_spatial_hierarchy", "lvl_spatial n node_parent") -_set("map_tec_addon", "t type_addon") -_set( - "map_tec_storage", - "n t m ts ms l c lvl_temporal", - description="Mapping of storage reservoir to charger/discharger", -) -_set("map_temporal_hierarchy", "lvl_temporal h time_parent") -_set("map_time", "time_parent h") -_set("type_tec_land", "type_tec") - -# Parameters -par("abs_cost_activity_soft_lo", "nl t ya h") -par("abs_cost_activity_soft_up", "nl t ya h") -par("abs_cost_new_capacity_soft_lo", "nl t yv") -par("abs_cost_new_capacity_soft_up", "nl t yv") -par("addon_conversion", "n t yv ya m h type_addon") -par("addon_lo", "n t ya m h type_addon") -par("addon_up", "n t ya m h type_addon") -par("bound_activity_lo", "nl t ya m h") -par("bound_activity_up", "nl t ya m h") -par("bound_emission", "n type_emission type_tec type_year") -par("bound_extraction_up", "n c g y") -par("bound_new_capacity_lo", "nl t yv") -par("bound_new_capacity_up", "nl t yv") -par("bound_total_capacity_lo", "nl t ya") -par("bound_total_capacity_up", "nl t ya") -par("capacity_factor", "nl t yv ya h") -par("commodity_stock", "n c l y") -par("construction_time", "nl t yv") -par("demand", "n c l y h") -par("duration_period", "y") -par("duration_time", "h") -par("dynamic_land_lo", "n s y u") -par("dynamic_land_up", "n s y u") -par("emission_factor", "nl t yv ya m e") -par("emission_scaling", "type_emission e") -par("fix_cost", "nl t yv ya") -par("fixed_activity", "nl t yv ya m h") -par("fixed_capacity", "nl t yv ya") -par("fixed_extraction", "n c g y") -par("fixed_land", "n s y") -par("fixed_new_capacity", "nl t yv") -par("fixed_stock", "n c l y") -par("flexibility_factor", "nl t yv ya m c l h q") -par("growth_activity_lo", "nl t ya h") -par("growth_activity_up", "nl t ya h") -par("growth_land_lo", "n y u") -par("growth_land_scen_lo", "n s y") -par("growth_land_scen_up", "n s y") -par("growth_land_up", "n y u") -par("growth_new_capacity_lo", "nl t yv") -par("growth_new_capacity_up", "nl t yv") -par("historical_activity", "nl t ya m h") -par("historical_emission", "n type_emission type_tec type_year") -par("historical_extraction", "n c g y") -par("historical_gdp", "n y") -par("historical_land", "n s y") -par("historical_new_capacity", "nl t yv") -par("initial_activity_lo", "nl t ya h") -par("initial_activity_up", "nl t ya h") -par("initial_land_lo", "n y u") -par("initial_land_scen_lo", "n s y") -par("initial_land_scen_up", "n s y") -par("initial_land_up", "n y u") -par("initial_new_capacity_lo", "nl t yv") -par("initial_new_capacity_up", "nl t yv") -par("input", "nl t yv ya m no c l h ho") -par("input_cap_new", "nl t yv no c l h") -par("input_cap_ret", "nl t yv no c l h") -par("input_cap", "nl t yv ya no c l h") -par("interestrate", "year") -par("inv_cost", "nl t yv") -par("land_cost", "n s y") -par("land_emission", "n s y e") -par("land_input", "n s y c l h") -par("land_output", "n s y c l h") -par("land_use", "n s y u") -par("level_cost_activity_soft_lo", "nl t ya h") -par("level_cost_activity_soft_up", "nl t ya h") -par("level_cost_new_capacity_soft_lo", "nl t yv") -par("level_cost_new_capacity_soft_up", "nl t yv") -par("min_utilization_factor", "nl t yv ya") -par("operation_factor", "nl t yv ya") -par("output_cap_new", "nl t yv nd c l h") -par("output_cap_ret", "nl t yv nd c l h") -par("output_cap", "nl t yv ya nd c l h") -par("output", "nl t yv ya m nd c l h hd") -par("peak_load_factor", "n c l y h") -par("rating_bin", "n t ya c l h q") -par("ref_activity", "nl t ya m h") -par("ref_extraction", "n c g y") -par("ref_new_capacity", "nl t yv") -par("ref_relation", "r nr yr") -par("relation_activity", "r nr yr nl t ya m") -par("relation_cost", "r nr yr") -par("relation_lower", "r nr yr") -par("relation_new_capacity", "r nr yr t") -par("relation_total_capacity", "r nr yr t") -par("relation_upper", "r nr yr") -par("reliability_factor", "n t ya c l h q") -par("renewable_capacity_factor", "n c g l y") -par("renewable_potential", "n c g l y") -par("resource_cost", "n c g y") -par("resource_remaining", "n c g y") -par("resource_volume", "n c g") -par("share_commodity_lo", "shares ns ya h") -par("share_commodity_up", "shares ns ya h") -par("share_mode_lo", "shares ns t m ya h") -par("share_mode_up", "shares ns t m ya h") -par("soft_activity_lo", "nl t ya h") -par("soft_activity_up", "nl t ya h") -par("soft_new_capacity_lo", "nl t yv") -par("soft_new_capacity_up", "nl t yv") -par("storage_initial", "n t m l c y h", "Initial amount of storage") -par( - "storage_self_discharge", - "n t m l c y h", - "Storage losses as a percentage of installed capacity", -) -par("subsidy", "nl type_tec ya") -par("tax_emission", "node type_emission type_tec type_year") -par("tax", "nl type_tec ya") -par("technical_lifetime", "nl t yv") -par("time_order", "lvl_temporal h", "Order of sub-annual time slices") -par("var_cost", "nl t yv ya m h") - -# Variables -var( - "ACT_LO", - "n t y h", - "Relaxation variable for dynamic constraints on activity (downwards)", -) -var( - "ACT_RATING", - "n t yv ya c l h q", - "Auxiliary variable for distributing total activity of a technology to a number of" - " 'rating bins'", -) -var( - "ACT_UP", - "n t y h", - "Relaxation variable for dynamic constraints on activity (upwards)", -) -var("ACT", "nl t yv ya m h", "Activity of technology") -var("CAP_FIRM", "n t c l y", "Capacity counting towards system reliability constraints") -var( - "CAP_NEW_LO", - "n t y", - "Relaxation variable for dynamic constraints on new capacity (downwards)", -) -var( - "CAP_NEW_UP", - "n t y", - "Relaxation variable for dynamic constraints on new capacity (upwards)", -) -var("CAP_NEW", "nl t yv", "New capacity") -var("CAP", "nl t yv ya", "Total installed capacity") -var("COMMODITY_BALANCE", "n c l y h", "Balance of commodity flow") -var( - "COMMODITY_USE", - "n c l y", - "Total amount of a commodity & level that was used or consumed", -) -var( - "COST_NODAL_NET", - "n y", - "System costs at the node level over time including effects of energy trade", -) -var("COST_NODAL", "n y", "System costs at the node level over time") -var("DEMAND", "n c l y h", "Demand") -var("EMISS", "n e type_tec y", "Aggregate emissions by technology type") -var("EXT", "n c g y", "Extraction of fossil resources") -var( - "GDP", - "n y", - "Gross domestic product (GDP) in market exchange rates for MACRO reporting", -) -var("LAND", "n s y", "Share of given land-use scenario") -var("OBJ", "", "Objective value of the optimisation problem (scalar)") -var( - "PRICE_COMMODITY", - "n c l y h", - "Commodity price (derived from marginals of COMMODITY_BALANCE constraint)", -) -var( - "PRICE_EMISSION", - "n type_emission type_tec y", - "Emission price (derived from marginals of EMISSION_EQUIVALENCE constraint)", -) -var( - "REL", - "r nr yr", - "Auxiliary variable for left-hand side of user-defined relations", -) -var( - "REN", - "n t c g y h", - "Activity of renewables specified per renewables grade", -) -var("SLACK_ACT_BOUND_LO", "n t y m h", "Slack variable for lower bound on activity") -var("SLACK_ACT_BOUND_UP", "n t y m h", "Slack variable for upper bound on activity") -var( - "SLACK_ACT_DYNAMIC_LO", - "n t y h", - "Slack variable for dynamic activity constraint relaxation (downwards)", -) -var( - "SLACK_ACT_DYNAMIC_UP", - "n t y h", - "Slack variable for dynamic activity constraint relaxation (upwards)", -) -var( - "SLACK_CAP_NEW_BOUND_LO", - "n t y", - "Slack variable for bound on new capacity (downwards)", -) -var( - "SLACK_CAP_NEW_BOUND_UP", - "n t y", - "Slack variable for bound on new capacity (upwards)", -) -var( - "SLACK_CAP_NEW_DYNAMIC_LO", - "n t y", - "Slack variable for dynamic new capacity constraint (downwards)", -) -var( - "SLACK_CAP_NEW_DYNAMIC_UP", - "n t y", - "Slack variable for dynamic new capacity constraint (upwards)", -) -var( - "SLACK_CAP_TOTAL_BOUND_LO", - "n t y", - "Slack variable for lower bound on total installed capacity", -) -var( - "SLACK_CAP_TOTAL_BOUND_UP", - "n t y", - "Slack variable for upper bound on total installed capacity", -) -var( - "SLACK_COMMODITY_EQUIVALENCE_LO", - "n c l y h", - "Slack variable for commodity balance (downwards)", -) -var( - "SLACK_COMMODITY_EQUIVALENCE_UP", - "n c l y h", - "Slack variable for commodity balance (upwards)", -) -var( - "SLACK_LAND_SCEN_LO", - "n s y", - "Slack variable for dynamic land scenario constraint relaxation (downwards)", -) -var( - "SLACK_LAND_SCEN_UP", - "n s y", - "Slack variable for dynamic land scenario constraint relaxation (upwards)", -) -var( - "SLACK_LAND_TYPE_LO", - "n y u", - "Slack variable for dynamic land type constraint relaxation (downwards)", -) -var( - "SLACK_LAND_TYPE_UP", - "n y u", - "Slack variable for dynamic land type constraint relaxation (upwards)", -) -var( - "SLACK_RELATION_BOUND_LO", - "r n y", - "Slack variable for lower bound of generic relation", -) -var( - "SLACK_RELATION_BOUND_UP", - "r n y", - "Slack variable for upper bound of generic relation", -) -var("STOCK_CHG", "n c l y h", "Annual input into and output from stocks of commodities") -var("STOCK", "n c l y", "Total quantity in intertemporal stock (storage)") -var( - "STORAGE_CHARGE", - "n t m l c y h", - "Charging of storage in each time slice (negative for discharge)", -) -var( - "STORAGE", - "n t m l c y h", - "State of charge (SoC) of storage at each sub-annual time slice (positive)", -) - -# Equations -equ( - "ACTIVITY_BOUND_ALL_MODES_LO", - "n t y h", - "Lower bound on activity summed over all vintages and modes", -) -equ( - "ACTIVITY_BOUND_ALL_MODES_UP", - "n t y h", - "Upper bound on activity summed over all vintages and modes", -) -equ( - "ACTIVITY_BOUND_LO", "n t y m h", "Lower bound on activity summed over all vintages" -) -equ( - "ACTIVITY_BOUND_UP", "n t y m h", "Upper bound on activity summed over all vintages" -) -equ( - "ACTIVITY_BY_RATING", - "n t y c l h q", - "Constraint on auxiliary rating-specific activity variable by rating bin", -) -equ( - "ACTIVITY_CONSTRAINT_LO", - "n t y h", - "Dynamic constraint on the market penetration of a technology activity" - " (lower bound)", -) -equ( - "ACTIVITY_CONSTRAINT_UP", - "n t y h", - "Dynamic constraint on the market penetration of a technology activity" - " (upper bound)", -) -equ("ACTIVITY_RATING_TOTAL", "n t yv y c l h", "Equivalence of `ACT_RATING` to `ACT`") -equ( - "ACTIVITY_SOFT_CONSTRAINT_LO", - "n t y h", - "Bound on relaxation of the dynamic constraint on market penetration (lower bound)", -) -equ( - "ACTIVITY_SOFT_CONSTRAINT_UP", - "n t y h", - "Bound on relaxation of the dynamic constraint on market penetration (upper bound)", -) -equ( - "ADDON_ACTIVITY_LO", - "n type_addon y m h", - "Addon technology activity lower constraint", -) -equ( - "ADDON_ACTIVITY_UP", - "n type_addon y m h", - "Addon-technology activity upper constraint", -) -# TODO I think inv_tec is defined only when writing out to GAMS, while this equation -# will need it to exist first -- but not populated, so just create it as a required set -# without data? -equ( - "CAPACITY_CONSTRAINT", - "n inv_tec yv y h", - "Capacity constraint for technology (by sub-annual time slice)", -) -equ( - "CAPACITY_MAINTENANCE_HIST", - "n inv_tec yv first_period", - "Constraint for capacity maintenance historical installation (built before " - "start of model horizon)", -) -equ( - "CAPACITY_MAINTENANCE_NEW", - "n inv_tec yv ya", - "Constraint for capacity maintenance of new capacity built in the current " - "period (vintage == year)", -) -equ( - "CAPACITY_MAINTENANCE", - "n inv_tec yv y", - "Constraint for capacity maintenance over the technical lifetime", -) -equ( - "COMMODITY_BALANCE_GT", - "n c l y h", - "Commodity supply greater than or equal demand", -) -equ("COMMODITY_BALANCE_LT", "n c l y h", "Commodity supply lower than or equal demand") -equ( - "COMMODITY_USE_LEVEL", - "n c l y h", - "Aggregate use of commodity by level as defined by total input into technologies", -) -equ("COST_ACCOUNTING_NODAL", "n y", "Cost accounting aggregated to the node") -equ( - "DYNAMIC_LAND_SCEN_CONSTRAINT_LO", - "n s y", - "Dynamic constraint on land scenario change (lower bound)", -) -equ( - "DYNAMIC_LAND_SCEN_CONSTRAINT_UP", - "n s y", - "Dynamic constraint on land scenario change (upper bound)", -) -equ( - "DYNAMIC_LAND_TYPE_CONSTRAINT_LO", - "n y u", - "Dynamic constraint on land-use change (lower bound)", -) -equ( - "DYNAMIC_LAND_TYPE_CONSTRAINT_UP", - "n y u", - "Dynamic constraint on land-use change (upper bound)", -) -equ( - "EMISSION_CONSTRAINT", - "n type_emission type_tec type_year", - "Nodal-regional-global constraints on emissions (by category)", -) -equ( - "EMISSION_EQUIVALENCE", - "n e type_tec y", - "Auxiliary equation to simplify the notation of emissions", -) -equ("EXTRACTION_BOUND_UP", "n c g y", "Upper bound on extraction (by grade)") -equ( - "EXTRACTION_EQUIVALENCE", - "n c y", - "Auxiliary equation to simplify the resource extraction formulation", -) -equ( - "FIRM_CAPACITY_PROVISION", - "n inv_tec y c l h", - "Contribution of dispatchable technologies to auxiliary firm-capacity variable", -) -equ( - "LAND_CONSTRAINT", - "n y", - "Constraint on total land use (partial sum of `LAND` on `land_scenario` is 1)", -) -equ( - "MIN_UTILIZATION_CONSTRAINT", - "n inv_tec yv y", - "Constraint for minimum yearly operation (aggregated over the course of a year)", -) -equ( - "NEW_CAPACITY_BOUND_LO", - "n inv_tec y", - "Lower bound on technology capacity investment", -) -equ( - "NEW_CAPACITY_BOUND_UP", - "n inv_tec y", - "Upper bound on technology capacity investment", -) -equ( - "NEW_CAPACITY_CONSTRAINT_LO", - "n inv_tec y", - "Dynamic constraint on capacity investment (lower bound)", -) -equ( - "NEW_CAPACITY_CONSTRAINT_UP", - "n inv_tec y", - "Dynamic constraint for capacity investment (learning and spillovers upper bound)", -) -equ( - "NEW_CAPACITY_SOFT_CONSTRAINT_LO", - "n inv_tec y", - "Bound on soft relaxation of dynamic new capacity constraints (downwards)", -) -equ( - "NEW_CAPACITY_SOFT_CONSTRAINT_UP", - "n inv_tec y", - "Bound on soft relaxation of dynamic new capacity constraints (upwards)", -) -equ("OBJECTIVE", "", "Objective value of the optimisation problem") -equ( - "OPERATION_CONSTRAINT", - "n inv_tec yv y", - "Constraint on maximum yearly operation (scheduled down-time for maintenance)", -) -equ("RELATION_CONSTRAINT_LO", "r n y", "Lower bound of relations (linear constraints)") -equ("RELATION_CONSTRAINT_UP", "r n y", "Upper bound of relations (linear constraints)") -equ( - "RELATION_EQUIVALENCE", - "r n y", - "Auxiliary equation to simplify the implementation of relations", -) -equ( - "RENEWABLES_CAPACITY_REQUIREMENT", - "n inv_tec c y", - "Lower bound on required overcapacity when using lower grade potentials", -) -equ( - "RENEWABLES_EQUIVALENCE", - "n renewable_tec c y h", - "Equation to define the renewables extraction", -) -equ( - "RENEWABLES_POTENTIAL_CONSTRAINT", - "n c g y", - "Constraint on renewable resource potential", -) -equ( - "RESOURCE_CONSTRAINT", - "n c g y", - "Constraint on resources remaining in each period (maximum extraction per period)", -) -equ( - "RESOURCE_HORIZON", - "n c g", - "Constraint on extraction over entire model horizon (resource volume in place)", -) -equ( - "SHARE_CONSTRAINT_COMMODITY_LO", - "shares ns y h", - "Lower bounds on share constraints for commodities", -) -equ( - "SHARE_CONSTRAINT_COMMODITY_UP", - "shares ns y h", - "Upper bounds on share constraints for commodities", -) -equ( - "SHARE_CONSTRAINT_MODE_LO", - "shares n t m y h", - "Lower bounds on share constraints for modes of a given technology", -) -equ( - "SHARE_CONSTRAINT_MODE_UP", - "shares n t m y h", - "Upper bounds on share constraints for modes of a given technology", -) -equ("STOCKS_BALANCE", "n c l y", "Commodity inter-temporal balance of stocks") -# FIXME 'h2' or 'time2' are not indicative names, come up with better ones -equ( - "STORAGE_BALANCE_INIT", - "n ts m l c y h h2", - "Balance of the state of charge of storage at sub-annual time slices with " - "initial storage content", -) -# TODO Why is this using h2 and not h? -equ( - "STORAGE_BALANCE", - "n ts m l c y h2 lvl_temporal", - "Balance of the state of charge of storage", -) -equ( - "STORAGE_CHANGE", - "n ts m level_storage c y h", - "Change in the state of charge of storage", -) -equ( - "STORAGE_INPUT", - "n ts l c level_storage c2 m y h", - "Connecting an input commodity to maintain the activity of storage container " - "(not stored commodity)", -) -equ( - "SYSTEM_FLEXIBILITY_CONSTRAINT", - "n c l y h", - "Constraint on total system flexibility", -) -equ( - "SYSTEM_RELIABILITY_CONSTRAINT", - "n c l y h", - "Constraint on total system reliability (firm capacity)", -) -equ("TOTAL_CAPACITY_BOUND_LO", "n inv_tec y", "Lower bound on total installed capacity") -equ("TOTAL_CAPACITY_BOUND_UP", "n inv_tec y", "Upper bound on total installed capacity") - - -#: ixmp items (sets, parameters, variables, and equations) for :class:`.MESSAGE`. -#: -#: .. deprecated:: 3.8.0 -#: Access the model class attribute :attr:`MESSAGE.items` instead. -MESSAGE_ITEMS = {k: v.to_dict() for k, v in MESSAGE.items.items()} - - -class MACRO(GAMSModel): - """Model class for MACRO.""" - - name = "MACRO" - - #: All equations, parameters, sets, and variables in the MACRO formulation. - items: MutableMapping[str, Item] = dict() - - #: MACRO uses the GAMS ``break;`` statement, and thus requires GAMS 24.8.1 or later. - GAMS_min_version = "24.8.1" - - keyword_to_solve_arg = [("concurrent", _boollike, "MACRO_CONCURRENT")] - - @classmethod - def initialize(cls, scenario, with_data=False): - """Initialize the model structure.""" - # NB some scenarios already have these items. This method simply adds any - # missing items. - - # Initialize the ixmp items for MACRO - cls.initialize_items(scenario, {k: v.to_dict() for k, v in cls.items.items()}) - - -equ = partial(_item_shorthand, MACRO, ItemType.EQU) -par = partial(_item_shorthand, MACRO, ItemType.PAR) -_set = partial(_item_shorthand, MACRO, ItemType.SET) -var = partial(_item_shorthand, MACRO, ItemType.VAR) - - -#: ixmp items (sets, parameters, variables, and equations) for MACRO. -_set("sector") -_set("mapping_macro_sector", "sector c l") -par("MERtoPPP", "n y") -par("aeei", "n sector y") -par("cost_MESSAGE", "n y") -par("demand_MESSAGE", "n sector y") -par("depr", "node") -par("drate", "node") -par("esub", "node") -par("gdp_calibrate", "n y") -par("grow", "n y") -par("historical_gdp", "n y") -par("kgdp", "node") -par("kpvs", "node") -par("lakl", "node") -par("lotol", "node") -par("prfconst", "n sector") -par("price_MESSAGE", "n sector y") -var("C", "n y", "Total consumption") -var("COST_NODAL", "n y") -var("COST_NODAL_NET", "n y", "Net of trade and emissions cost") -var("DEMAND", "n c l y h") -var("EC", "n y") -var("GDP", "n y") -var("I", "n y", "Total investment") -var("K", "n y") -var("KN", "n y") -var("MAX_ITER", "") -var("N_ITER", "") -var("NEWENE", "n sector y") -var("PHYSENE", "n sector y") -var("PRICE", "n c l y h") -var("PRODENE", "n sector y") -var("UTILITY", "") -var("Y", "n y") -var("YN", "n y") -var("aeei_calibrate", "n sector y") -var("grow_calibrate", "n y") -equ("COST_ACCOUNTING_NODAL", "n y") - -#: ixmp items (sets, parameters, variables, and equations) for :class:`.MACRO`. -#: -#: .. deprecated:: 3.8.0 -#: Access the model class attribute :attr:`MACRO.items` instead. -MACRO_ITEMS = {k: v.to_dict() for k, v in MACRO.items.items()} - - -class MESSAGE_MACRO(MESSAGE, MACRO): - """Model class for MESSAGE_MACRO.""" - - name = "MESSAGE-MACRO" - #: All equations, parameters, sets, and variables in the MESSAGE-MACRO formulation. - items = ChainMap(MESSAGE.items, MACRO.items) - - keyword_to_solve_arg = ( - MACRO.keyword_to_solve_arg - + MESSAGE.keyword_to_solve_arg - + [ - ("convergence_criterion", float, "CONVERGENCE_CRITERION"), - ("max_adjustment", float, "MAX_ADJUSTMENT"), - ("max_iteration", int, "MAX_ITERATION"), - ] + submodule = "message_ix." + _DEPRECATED_IMPORT[name] + except KeyError: + raise AttributeError(name) + + warn( + f"from message_ix.models import {name}\n" + f"Instead, use:\n from {submodule} import {name}", + DeprecationWarning, + stacklevel=2, ) - @classmethod - def initialize(cls, scenario, with_data=False): - MESSAGE.initialize(scenario) - MACRO.initialize(scenario, with_data) + return getattr(import_module(submodule), name) diff --git a/message_ix/report/__init__.py b/message_ix/report/__init__.py index c3b602e43..931106fda 100644 --- a/message_ix/report/__init__.py +++ b/message_ix/report/__init__.py @@ -15,7 +15,7 @@ ) from ixmp.report import Reporter as IXMPReporter -from message_ix.models import DIMS +from message_ix.common import DIMS from .pyam import collapse_message_cols diff --git a/message_ix/testing/__init__.py b/message_ix/testing/__init__.py index a1e5e5f65..e88d0dde9 100644 --- a/message_ix/testing/__init__.py +++ b/message_ix/testing/__init__.py @@ -34,10 +34,10 @@ def pytest_report_header(config: pytest.Config, start_path: Path) -> str: def pytest_sessionstart() -> None: """Use only 2 threads for CPLEX on GitHub Actions runners with 2 CPU cores.""" - import message_ix.models + import message_ix.common if "GITHUB_ACTIONS" in os.environ: - message_ix.models.DEFAULT_CPLEX_OPTIONS["threads"] = 2 + message_ix.common.DEFAULT_CPLEX_OPTIONS["threads"] = 2 # Data for testing @@ -928,7 +928,7 @@ def snapshots_from_zenodo( """ from pooch import HTTPDownloader, Pooch - from message_ix.models import MACRO + from message_ix.macro import MACRO # Create a pooch 'registry' from `SCENARIOS` assert pytestconfig.cache, "Pytest config does not have `.cache` set!" diff --git a/message_ix/tests/test_feature_storage.py b/message_ix/tests/test_feature_storage.py index 59ed2b38b..86357d53a 100644 --- a/message_ix/tests/test_feature_storage.py +++ b/message_ix/tests/test_feature_storage.py @@ -20,7 +20,7 @@ from ixmp.testing import assert_logs from message_ix import Scenario -from message_ix.models import MESSAGE +from message_ix.message import MESSAGE from message_ix.testing import make_dantzig from message_ix.util import expand_dims diff --git a/message_ix/tests/test_macro.py b/message_ix/tests/test_macro.py index 2c2601607..e3b198b02 100644 --- a/message_ix/tests/test_macro.py +++ b/message_ix/tests/test_macro.py @@ -8,17 +8,18 @@ import pytest from ixmp import Platform -from message_ix import Scenario, macro -from message_ix.models import MACRO +from message_ix import Scenario +from message_ix.macro import MACRO +from message_ix.macro.calibrate import add_model_data, calibrate, prepare_computer from message_ix.report import Quantity from message_ix.testing import SCENARIO, make_westeros # NOTE These tests maybe don't need to be parametrized # Do the following depend on otherwise untested Scenario functions? # Scenario.add_macro() -# macro.prepare_computer() -# macro.add_model_data() -# macro.calibrate() +# macro.calibrate.prepare_computer() +# macro.calibrate.add_model_data() +# macro.calibrate.calibrate() # MACRO.initialize() @@ -70,25 +71,23 @@ def westeros_not_solved( def test_calc_valid_data_file(westeros_solved: Scenario, w_data_path: Path) -> None: - c = macro.prepare_computer(westeros_solved, data=w_data_path) + c = prepare_computer(westeros_solved, data=w_data_path) c.get("check all") def test_calc_invalid_data(westeros_solved: Scenario) -> None: # TypeError is raised with invalid input data type with pytest.raises(TypeError, match="neither a dict nor a valid path"): - macro.prepare_computer(westeros_solved, data=list()) # type: ignore [arg-type] + prepare_computer(westeros_solved, data=list()) # type: ignore [arg-type] with pytest.raises(ValueError, match="not an Excel data file"): - macro.prepare_computer( - westeros_solved, data=Path(__file__).joinpath("other.zip") - ) + prepare_computer(westeros_solved, data=Path(__file__).joinpath("other.zip")) def test_calc_valid_data_dict( westeros_solved: Scenario, w_data: dict[str, pd.DataFrame] ) -> None: - c = macro.prepare_computer(westeros_solved, data=w_data) + c = prepare_computer(westeros_solved, data=w_data) c.get("check all") @@ -117,11 +116,11 @@ def test_calc_valid_years( def test_calc_no_solution(westeros_not_solved: Scenario, w_data_path: Path) -> None: s = westeros_not_solved with pytest.raises(RuntimeError, match="solution"): - macro.prepare_computer(s, data=w_data_path) + prepare_computer(s, data=w_data_path) def test_config(westeros_solved: Scenario, w_data_path: Path) -> None: - c = macro.prepare_computer(westeros_solved, data=w_data_path) + c = prepare_computer(westeros_solved, data=w_data_path) assert "config::macro" in c.graph assert "sector" in c.get("config::macro") assert {"Westeros"} == c.get("node::macro") @@ -130,13 +129,13 @@ def test_config(westeros_solved: Scenario, w_data_path: Path) -> None: # Missing columns in config raises an exception data = c.get("data") data["config"] = data["config"][["node", "sector", "commodity", "year"]] - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) with pytest.raises(Exception, match="level"): c.get("check all") # Entirely missing config raises an exception data.pop("config") - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) with pytest.raises(Exception, match="config"): c.get("check all") @@ -148,7 +147,7 @@ def test_calc_data_missing_par( data.pop("gdp_calibrate") - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) with pytest.raises(Exception, match="gdp_calibrate"): c.get("check all") @@ -165,7 +164,7 @@ def test_calc_data_missing_ref( data.pop("price_ref") - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) c.get("check all") westeros_solved.add_macro(data) @@ -179,7 +178,7 @@ def test_calc_data_missing_column( # Drop a column data["gdp_calibrate"] = data["gdp_calibrate"].drop("year", axis=1) - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) with pytest.raises(Exception, match="Missing expected columns.*year"): c.get("check all") @@ -192,7 +191,7 @@ def test_calc_data_missing_datapoint( # Skip first data point data["gdp_calibrate"] = data["gdp_calibrate"][1:] - c = macro.prepare_computer(westeros_solved, data=data) + c = prepare_computer(westeros_solved, data=data) with pytest.raises(Exception, match="Must provide two gdp_calibrate data points"): c.get("check all") @@ -231,7 +230,7 @@ def test_calc( expected: Union[list[float], list[int]], ) -> None: """Test calculation of intermediate values on a solved Westeros scenario.""" - c = macro.prepare_computer(westeros_solved, data=w_data_path) + c = prepare_computer(westeros_solved, data=w_data_path) assertion = getattr(npt, f"assert_{test}") @@ -257,7 +256,7 @@ def test_calc_price_zero(westeros_not_solved: Scenario, w_data_path: Path) -> No pc = s.var("PRICE_COMMODITY") assert np.isclose(0, pc["lvl"]).any(), f"No zero values in:\n{pc.to_string()}" - c = macro.prepare_computer(s, data=w_data_path) + c = prepare_computer(s, data=w_data_path) with pytest.raises(Exception, match="0-price found in MESSAGE variable PRICE_"): c.get("price_MESSAGE") @@ -283,7 +282,7 @@ def test_add_model_data(westeros_solved: Scenario, w_data_path: Path) -> None: clone = base.clone(scenario=f"{base.scenario} cloned", keep_solution=False) clone.check_out() MACRO.initialize(clone) - macro.add_model_data(base, clone, w_data_path) + add_model_data(base, clone, w_data_path) clone.commit("finished adding macro") clone.solve(quiet=True) obs = clone.var("OBJ")["lvl"] @@ -297,13 +296,13 @@ def test_calibrate(westeros_solved: Scenario, w_data_path: Path) -> None: clone = base.clone(base.model, "test macro calibration", keep_solution=False) clone.check_out() MACRO.initialize(clone) - macro.add_model_data(base, clone, w_data_path) + add_model_data(base, clone, w_data_path) clone.commit("finished adding macro") start_aeei = clone.par("aeei")["value"] start_grow = clone.par("grow")["value"] - macro.calibrate(clone, check_convergence=True) + calibrate(clone, check_convergence=True) end_aeei = clone.par("aeei")["value"] end_grow = clone.par("grow")["value"] @@ -364,14 +363,14 @@ def mr_scenario( def test_multiregion_valid_data(mr_scenario: Scenario) -> None: """Multi-region, multi-sector input data can be checked.""" s = mr_scenario - c = macro.prepare_computer(s, data=mr_data_path("2")) + c = prepare_computer(s, data=mr_data_path("2")) c.get("check all") def test_multiregion_derive_data(mr_scenario: Scenario) -> None: s = mr_scenario path = mr_data_path("1") - c = macro.prepare_computer(s, data=path) + c = prepare_computer(s, data=path) # Fake some data; this emulates the behaviour of the MockScenario class formerly # used in this file @@ -414,7 +413,7 @@ def test_multiregion_derive_data(mr_scenario: Scenario) -> None: def test_multiregion_derive_data_2(mr_scenario: Scenario) -> None: """Multi-region multi-sector data can be computed.""" s = mr_scenario - c = macro.prepare_computer(s, data=mr_data_path("2")) + c = prepare_computer(s, data=mr_data_path("2")) c.get("check all") diff --git a/message_ix/tests/test_message.py b/message_ix/tests/test_message.py new file mode 100644 index 000000000..e3e65b310 --- /dev/null +++ b/message_ix/tests/test_message.py @@ -0,0 +1,74 @@ +import re +from collections import defaultdict +from typing import TYPE_CHECKING + +import ixmp +from ixmp.backend.jdbc import JDBCBackend + +from message_ix.message import MESSAGE + +if TYPE_CHECKING: + from ixmp import Platform + + +class TestMESSAGE: + """Tests of :class:`.MESSAGE`.""" + + def test_initialize(self, caplog, test_mp: "Platform") -> None: + # Expected numbers of items by type + exp = defaultdict(list) + for name, spec in MESSAGE.items.items(): + exp[str(spec.type.name).lower()].append(name) + + # balance_equality is removed in initialize() for JDBC + if isinstance(test_mp._backend, JDBCBackend): + exp["set"].remove("balance_equality") + + # Use ixmp.Scenario to avoid invoking ixmp_source/Java code that automatically + # populates empty scenarios + s = ixmp.Scenario(test_mp, model="m", scenario="s", version="new") + + # Initialization succeeds on a totally empty scenario + MESSAGE.initialize(s) + + # The expected items exist + for ix_type, exp_names in exp.items(): + obs_names = getattr(s, f"{ix_type}_list")() + assert sorted(obs_names) == sorted(exp_names) + + def test_initialize_filter_log(self, caplog, test_mp: "Platform") -> None: + """Test :meth:`MESSAGE.initialize` logging under some conditions. + + For :class:`.Scenario` created with message_ix v3.10 or earlier, equations and + variables may be initialized but have zero dimensions, thus empty lists of + "index sets" and "index names". When :class:`.Scenario` is instantiated, + :meth:`MESSAGE.initialize` is invoked, and in turn + :meth:`ixmp.model.base.Model.initialize_items`. This method generates many log + messages on level :data:`~logging.WARNING`. + + In order to prevent this log noise, :func:`.models._filter_log_initialize_items` + is used. This test checks that it is effective. + """ + # Use ixmp.Scenario to avoid invoking ixmp_source/Java code that automatically + # populates empty scenarios + s = ixmp.Scenario(test_mp, model="m", scenario="s", version="new") + + # Initialize an equation with no dimensions. This mocks the state of a Scenario + # created with message_ix v3.10 or earlier. + s.init_equ("NEW_CAPACITY_BOUND_LO", idx_sets=[], idx_names=[]) + + s.commit("") + s.set_as_default() + caplog.clear() + + # Initialize items. + MESSAGE.initialize(s) + + # Messages related to re-initializing items with 0 dimensions are filtered and + # do not reach `caplog`. This assertion fails with message_ix v3.10. + message_pattern = re.compile( + r"Existing index (name|set)s of 'NEW_CAPACITY_BOUND_LO' \[\] do not match " + r"\('node', '.*', 'year'\)" + ) + extra = list(filter(message_pattern.match, caplog.messages)) + assert not extra, f"{len(extra)} unwanted log messages: {extra}" diff --git a/message_ix/tests/test_message_macro.py b/message_ix/tests/test_message_macro.py new file mode 100644 index 000000000..f606f8ea0 --- /dev/null +++ b/message_ix/tests/test_message_macro.py @@ -0,0 +1,52 @@ +import pytest +from ixmp import Platform + +from message_ix import Scenario +from message_ix.message_macro import MESSAGE_MACRO + + +class TestMESSAGE_MACRO: + """Tests of :class:`.MESSAGE_MACRO`.""" + + @pytest.mark.parametrize( + "kwargs, solve_args", + ( + (dict(), []), + ( + dict(convergence_criterion=0.02, max_adjustment=0.4, max_iteration=100), + [ + "--CONVERGENCE_CRITERION=0.02", + "--MAX_ADJUSTMENT=0.4", + "--MAX_ITERATION=100", + ], + ), + ), + ) + def test_init(self, kwargs, solve_args) -> None: + # A model instance can be constructed with the given `kwargs` + mm = MESSAGE_MACRO(**kwargs) + + # Command-line options to GAMS have expected form + assert all(e in mm.solve_args for e in solve_args), mm.solve_args + + def test_init_GAMS_min_version(self) -> None: + class _MM(MESSAGE_MACRO): + """Dummy subclass requiring a non-existent GAMS version.""" + + GAMS_min_version = "999.9.9" + + # Constructor complains about an insufficient GAMS version + with pytest.raises( + RuntimeError, match="MESSAGE-MACRO requires GAMS >= 999.9.9; got " + ): + _MM() + + def test_initialize(self, test_mp: Platform) -> None: + # MESSAGE_MACRO.initialize() runs + Scenario( + test_mp, + "test_message_macro", + "test_initialize", + version="new", + scheme="MESSAGE-MACRO", + ) diff --git a/message_ix/tests/test_models.py b/message_ix/tests/test_models.py index d6e2b367f..7f79be583 100644 --- a/message_ix/tests/test_models.py +++ b/message_ix/tests/test_models.py @@ -1,104 +1,29 @@ -"""Tests of :class:`ixmp.model.base.Model` classes in :mod:`message_ix.models`.""" +from contextlib import nullcontext -import re -from collections import defaultdict -from typing import TYPE_CHECKING - -import ixmp import pytest -from ixmp.backend.jdbc import JDBCBackend - -from message_ix.models import MESSAGE, MESSAGE_MACRO - -if TYPE_CHECKING: - from ixmp import Platform - - -class TestMESSAGE: - def test_initialize(self, caplog, test_mp: "Platform") -> None: - # Expected numbers of items by type - exp = defaultdict(list) - for name, spec in MESSAGE.items.items(): - exp[str(spec.type.name).lower()].append(name) - - # balance_equality is removed in initialize() for JDBC - if isinstance(test_mp._backend, JDBCBackend): - exp["set"].remove("balance_equality") - - # Use ixmp.Scenario to avoid invoking ixmp_source/Java code that automatically - # populates empty scenarios - s = ixmp.Scenario(test_mp, model="m", scenario="s", version="new") - - # Initialization succeeds on a totally empty scenario - MESSAGE.initialize(s) - - # The expected items exist - for ix_type, exp_names in exp.items(): - obs_names = getattr(s, f"{ix_type}_list")() - assert sorted(obs_names) == sorted(exp_names) - - def test_initialize_filter_log(self, caplog, test_mp: "Platform") -> None: - """Test :meth:`MESSAGE.initialize` logging under some conditions. - For :class:`.Scenario` created with message_ix v3.10 or earlier, equations and - variables may be initialized but have zero dimensions, thus empty lists of - "index sets" and "index names". When :class:`.Scenario` is instantiated, - :meth:`MESSAGE.initialize` is invoked, and in turn - :meth:`ixmp.model.base.Model.initialize_items`. This method generates many log - messages on level :data:`~logging.WARNING`. - In order to prevent this log noise, :func:`.models._filter_log_initialize_items` - is used. This test checks that it is effective. - """ - # Use ixmp.Scenario to avoid invoking ixmp_source/Java code that automatically - # populates empty scenarios - s = ixmp.Scenario(test_mp, model="m", scenario="s", version="new") - - # Initialize an equation with no dimensions. This mocks the state of a Scenario - # created with message_ix v3.10 or earlier. - s.init_equ("NEW_CAPACITY_BOUND_LO", idx_sets=[], idx_names=[]) - - s.commit("") - s.set_as_default() - caplog.clear() - - # Initialize items. - MESSAGE.initialize(s) - - # Messages related to re-initializing items with 0 dimensions are filtered and - # do not reach `caplog`. This assertion fails with message_ix v3.10. - message_pattern = re.compile( - r"Existing index (name|set)s of 'NEW_CAPACITY_BOUND_LO' \[\] do not match " - r"\('node', '.*', 'year'\)" +@pytest.mark.parametrize( + "name", + ( + "DIMS", + "Item", + "MACRO", + "MESSAGE", + "MESSAGE_MACRO", + pytest.param("FOO", marks=pytest.mark.xfail(raises=AttributeError)), + ), +) +def test_deprecated_import(name: str) -> None: + import message_ix.models + + ctx = ( + nullcontext() + if name == "FOO" + else pytest.warns( + DeprecationWarning, match=f"from message_ix.models import {name}" ) - extra = list(filter(message_pattern.match, caplog.messages)) - assert not extra, f"{len(extra)} unwanted log messages: {extra}" - - -def test_message_macro() -> None: - # Constructor runs successfully - MESSAGE_MACRO() - - class _MM(MESSAGE_MACRO): - """Dummy subclass requiring a non-existent GAMS version.""" - - GAMS_min_version = "99.9.9" - - # Constructor complains about an insufficient GAMS version - with pytest.raises( - RuntimeError, match="MESSAGE-MACRO requires GAMS >= 99.9.9; got " - ): - _MM() - - # Construct with custom model options - mm = MESSAGE_MACRO( - convergence_criterion=0.02, max_adjustment=0.4, max_iteration=100 ) - # Command-line options to GAMS have expected form - expected = [ - "--CONVERGENCE_CRITERION=0.02", - "--MAX_ADJUSTMENT=0.4", - "--MAX_ITERATION=100", - ] - assert all(e in mm.solve_args for e in expected) + with ctx: + getattr(message_ix.models, name) diff --git a/message_ix/tests/test_report.py b/message_ix/tests/test_report.py index 5142c6d45..bc62cd5c6 100644 --- a/message_ix/tests/test_report.py +++ b/message_ix/tests/test_report.py @@ -21,7 +21,7 @@ from pandas.testing import assert_frame_equal, assert_series_equal from message_ix import Scenario -from message_ix.models import MESSAGE +from message_ix.message import MESSAGE from message_ix.report import Reporter, configure from message_ix.testing import SCENARIO, make_dantzig, make_westeros diff --git a/message_ix/util/__init__.py b/message_ix/util/__init__.py index 0986f27b2..06683aa5e 100644 --- a/message_ix/util/__init__.py +++ b/message_ix/util/__init__.py @@ -10,7 +10,8 @@ from ixmp.backend import ItemType from pandas.api.types import is_scalar -from message_ix.models import MACRO, MESSAGE +from message_ix.macro import MACRO +from message_ix.message import MESSAGE if TYPE_CHECKING: from message_ix.core import Scenario