Dedicated us inputs scenarios (#181)

* code: introduce dedicated inputs and folders for the US

* code: add manual input usa

* code: latest changes to compile_cost_assumptions_usa.py

* code: latest changes to compile_cost_assumptions_usa.py - 2

* code: relocate adjust_for_inflation to _helpers.py

* code: update costs in manual_input.csv

* code: next step on compile_cost_assumptions_usa.py

* code: updates

* code: add docstring

* code: pre-commit

* code: new updates to docstring

* code: docstring

* code: docstring in _helpers.py

* docu: update the release notes

* code: add unit tests

* code: add pre-commit

* code: sort outputs

* code: add pre-commit

* doc: add docstring

* code: re-import numpy and pandas in _helpers.py

* add scenarios for H2 and DAC (US only) + apply inflation rate to EUR only

* doc: update release notes

* code: fix unit tests

* data: modify manual_input_usa.csv specifying financial_case R&D for rows where scenario is defined

* add missing scenarios for electrolyzers

* pre-commit changes

* reorder manual_input_usa.csv

* add new files

* fix financial case for ICCT technologies

* ensure that financial case is correctly reported in output files

* update unit test

* modify representative NREL technologies

* include description of scenario and financial case

* code: resolve FutureWarning from set_specify_assumptions

---------

Co-authored-by: Fabrizio Finozzi <[email protected]>
Co-authored-by: Fabrizio Finozzi <[email protected]>

Author: danielelerede-oet

config.yaml

@@ -13,6 +13,7 @@ nrel_atb:
   nrel_atb_core_metric_parameter_to_keep: ["CAPEX", "CF", "Fixed O&M", "Variable O&M", "Fuel", "Additional OCC"]
   nrel_atb_technology_to_remove: ["Coal-CCS-95% -> Transformational Tech", "Coal-Max-CCS -> Transformational Tech", "Coal-new -> Transformational Tech", "NG combined cycle 95% CCS (F-frame basis -> Transformational Tech)", "NG combined cycle 95% CCS (H-frame basis -> Transformational Tech)", "NG combined cycle Max CCS (F-frame basis -> Transformational Tech)", "NG combined cycle Max CCS (H-frame basis -> Transformational Tech)", "AEO"]
   nrel_atb_source_link: "NREL/ATB-https://data.openei.org/s3_viewer?bucket=oedi-data-lake&prefix=ATB%2Felectricity%2Fcsv%2F2022%2F"
+  nrel_atb_further_description: "Meaning of scenario and financial case: https://atb.nrel.gov/electricity/2024/definitions#scenarios"
 
 expectation: ""  # tech data uncertainty, possible options [None, "optimist", "pessimist"]
 

docs/release_notes.rst

@@ -24,9 +24,11 @@ Upcoming Release
 
 * Align `snakemake` version and the related `mock_snakemake` to PyPSA-Eur (https://github.com/PyPSA/technology-data/pull/177)
 
+* Improve filename consistency in the sources (https://github.com/PyPSA/technology-data/pull/178)
+
 * Improve assumptions for iron-air batteries (https://github.com/PyPSA/technology-data/pull/179)
 
-* Improve filename consistency in the sources (https://github.com/PyPSA/technology-data/pull/178)
+* US-specific scenarios for electrolyzers and DAC + adjustment for inflation removed as already considered in input data (https://github.com/PyPSA/technology-data/pull/181)
 
 * Include further unit tests for compile_cost_assumptions_usa.py (https://github.com/PyPSA/technology-data/pull/182)
 

docs/structure.rst

@@ -49,6 +49,7 @@ This repository has the following structure:
     * nrel_atb_core_metric_parameter_to_keep : list of parameters to use from the NREL/ATB source files
     * nrel_atb_technology_to_remove : list of technologies that should be excluded from NREL/ATB
     * nrel_atb_source_link : source url for the NREL/ATB data used
+    * nrel_atb_further_description : Meaning of "scenario" and "financial case"
     * expectation : tech data uncertainty, possible options [None, "optimist", "pessimist"]
     * eur_year : year for EUR outputs
     * solar_utility_from_vartiaien : Bool (True/False) if solar utility data is taken from DEA or Vartiaien

inputs/US/discount_rates_usa.csv

@@ -60,10 +60,10 @@ def get_conversion_dictionary(flag: str) -> dict:
             "Nuclear - Large": "nuclear",
             "Nuclear - AP1000": "nuclear",
             "Geothermal - Hydro / Flash": "geothermal",
-            "Land-Based Wind - Class 1": "onwind",
-            "Land-Based Wind - Class 1 - Technology 1": "onwind",
-            "Offshore Wind - Class 1": "offwind",
-            "Utility PV - Class 1": "solar-utility",
+            "Land-Based Wind - Class 4": "onwind",
+            "Land-Based Wind - Class 4 - Technology 1": "onwind",
+            "Offshore Wind - Class 3": "offwind",
+            "Utility PV - Class 5": "solar-utility",
             "Commercial PV - Class 1": "solar-rooftop",
             "Utility-Scale Battery Storage - 6Hr": "battery storage",
             "Biopower": "biomass",
@@ -72,9 +72,9 @@ def get_conversion_dictionary(flag: str) -> dict:
         }
     elif flag.casefold() == "atb_technology_name":
         return {
+            "Land-Based Wind - Class 1": "Land-Based Wind - Class 1 - Technology 1",
             "Land-Based Wind - Class 2": "Land-Based Wind - Class 2 - Technology 1",
             "Land-Based Wind - Class 3": "Land-Based Wind - Class 3 - Technology 1",
-            "Land-Based Wind - Class 4": "Land-Based Wind - Class 4 - Technology 1",
             "Land-Based Wind - Class 5": "Land-Based Wind - Class 5 - Technology 1",
             "Land-Based Wind - Class 6": "Land-Based Wind - Class 6 - Technology 1",
             "Land-Based Wind - Class 7": "Land-Based Wind - Class 7 - Technology 1",
@@ -357,23 +357,65 @@ def pre_process_manual_input_usa(
                 "technology == @tech and parameter == @param"
             )
 
-            s = pd.Series(
-                index=list_of_years,
-                data=np.interp(list_of_years, c["year"], c["value"]),
-                name=param,
-            )
-            s["parameter"] = param
-            s["technology"] = tech
-            try:
-                s["currency_year"] = int(c["currency_year"].values[0])
-            except ValueError:
-                s["currency_year"] = np.nan
-            for col in ["unit", "source", "further description"]:
-                s[col] = "; and\n".join(c[col].unique().astype(str))
-            s = s.rename(
-                {"further_description": "further description"}
-            )  # match column name between manual_input and original TD workflow
-            list_dataframe_row.append(s)
+            # Consider differences among scenarios
+            scenarios = c["scenario"].dropna().unique()
+
+            for scenario in scenarios:
+                scenario_value = c[c["scenario"] == scenario][
+                    "value"
+                ].values  # Extract values for each scenario
+
+                if scenario_value.size > 0:
+                    scenario_years = c[c["scenario"] == scenario]["year"].values
+                    scenario_values = c[c["scenario"] == scenario]["value"].values
+
+                    interpolated_values = np.interp(
+                        list_of_years, scenario_years, scenario_values
+                    )
+
+                    # Create a row for each scenario
+                    s_copy = pd.Series(
+                        index=list_of_years,
+                        data=interpolated_values,  # values are now interpolated
+                        name=param,
+                    )
+
+                    s_copy["parameter"] = param
+                    s_copy["technology"] = tech
+                    s_copy["scenario"] = scenario
+                    try:
+                        s_copy["currency_year"] = int(c["currency_year"].values[0])
+                    except ValueError:
+                        s_copy["currency_year"] = np.nan
+
+                    # Add the other columns in the data file
+                    for col in ["unit", "source", "further description"]:
+                        s_copy[col] = c[col].unique()[0]
+
+                    # Add a separate row for each `financial_case`
+                    for financial_case in c["financial_case"].unique():
+                        s_copy["financial_case"] = financial_case
+                        list_dataframe_row.append(s_copy.copy())
+            if len(scenarios) == 0:
+                s = pd.Series(
+                    index=list_of_years,
+                    data=[scenario_value] * len(list_of_years),
+                    name=param,
+                )
+                s["parameter"] = param
+                s["technology"] = tech
+                s["scenario"] = ""
+                try:
+                    s["currency_year"] = int(c["currency_year"].values[0])
+                except ValueError:
+                    s["currency_year"] = np.nan
+                for col in ["unit", "source", "further description"]:
+                    s[col] = c[col].unique()[0]
+
+                # Add a separate row for each `financial_case`
+                for financial_case in c["financial_case"].unique():
+                    s["financial_case"] = financial_case
+                    list_dataframe_row.append(s.copy())
     manual_input_usa_file_df = pd.DataFrame(list_dataframe_row).reset_index(drop=True)
 
     # Filter the information for a given year
@@ -386,25 +428,56 @@ def pre_process_manual_input_usa(
             "source",
             "further description",
             "currency_year",
+            "financial_case",
+            "scenario",
         ]
     ].rename(columns={year: "value"})
 
+    # Filter data to get technologies with scenario differentiation
+    with_scenario_df = manual_input_usa_file_df[
+        manual_input_usa_file_df["scenario"].notna()
+    ]
+    without_scenario_df = manual_input_usa_file_df[
+        manual_input_usa_file_df["scenario"].isna()
+    ]
+
+    final_rows = []
+
+    for tech in manual_input_usa_file_df["technology"].unique():
+        tech_with_scenario = with_scenario_df[with_scenario_df["technology"] == tech]
+        if len(tech_with_scenario) > 0:
+            # Keep rows where a scenario exists
+            final_rows.append(tech_with_scenario)
+        else:
+            # If a scenario is not defined, keep the row without scenario
+            tech_without_scenario = without_scenario_df[
+                without_scenario_df["technology"] == tech
+            ]
+            final_rows.append(tech_without_scenario)
+
+    manual_input_usa_file_df = pd.concat(final_rows, ignore_index=True)
+
     # Cast the value column to float
     manual_input_usa_file_df["value"] = manual_input_usa_file_df["value"].astype(float)
 
     # Correct the cost assumptions to the inflation rate
-    inflation_adjusted_manual_input_usa_file_df = adjust_for_inflation(
-        inflation_rate_series,
-        manual_input_usa_file_df,
-        manual_input_usa_file_df.technology.unique(),
-        eur_year,
-        "value",
-        usa_costs_flag=True,
+    mask = manual_input_usa_file_df["unit"].str.startswith("EUR", na=False)
+
+    inflation_adjusted_manual_input_usa_file_df = manual_input_usa_file_df.copy()
+    inflation_adjusted_manual_input_usa_file_df.loc[mask, "value"] = (
+        adjust_for_inflation(
+            inflation_rate_series,
+            manual_input_usa_file_df.loc[mask],
+            manual_input_usa_file_df.loc[mask, "technology"].unique(),
+            eur_year,
+            "value",
+            usa_costs_flag=True,
+        )["value"]
     )
 
     # Round the results
     inflation_adjusted_manual_input_usa_file_df.loc[:, "value"] = round(
-        inflation_adjusted_manual_input_usa_file_df.value.astype(float), n_digits
+        inflation_adjusted_manual_input_usa_file_df["value"].astype(float), n_digits
     )
 
     return inflation_adjusted_manual_input_usa_file_df
@@ -688,6 +761,7 @@ def pre_process_cost_input_file(
 def pre_process_atb_input_file(
     input_file_path: str,
     nrel_source: str,
+    nrel_further_description: str,
     year: int,
     list_columns_to_keep: list,
     list_core_metric_parameter_to_keep: list,
@@ -803,7 +877,7 @@ def pre_process_atb_input_file(
     atb_input_df["source"] = nrel_source
 
     # Add further description column
-    atb_input_df["further description"] = pd.Series(dtype="str")
+    atb_input_df["further description"] = nrel_further_description
 
     # Rename columns and select just columns used in PyPSA
     column_rename_dict = get_conversion_dictionary("output_column")
@@ -923,6 +997,9 @@ def duplicate_fuel_cost(input_file_path: str, list_of_years: list) -> pd.DataFra
         "nrel_atb_core_metric_parameter_to_keep"
     ]
     nrel_atb_source_link = snakemake.config["nrel_atb"]["nrel_atb_source_link"]
+    nrel_atb_further_description = snakemake.config["nrel_atb"][
+        "nrel_atb_further_description"
+    ]
     nrel_atb_technology_to_remove = snakemake.config["nrel_atb"][
         "nrel_atb_technology_to_remove"
     ]
@@ -989,6 +1066,7 @@ def duplicate_fuel_cost(input_file_path: str, list_of_years: list) -> pd.DataFra
         atb_e_df = pre_process_atb_input_file(
             input_atb_path,
             nrel_atb_source_link,
+            nrel_atb_further_description,
             year_val,
             nrel_atb_columns_to_keep,
             nrel_atb_core_metric_parameter_to_keep,

inputs/US/manual_input_usa.csv

@@ -288,6 +288,7 @@ def test_set_specify_assumptions():
                 "decentral gas boiler",
                 "decentral gas boiler",
                 "biogas upgrading",
+                "biogas upgrading",
                 "solar-rooftop",
                 "heat pump",
             ],
@@ -297,34 +298,37 @@ def test_set_specify_assumptions():
                 "Possible additional specific investment",
                 "Technical lifetime",
                 "investment",
+                "Technical lifetime",
                 "PV module conversion efficiency [p.u.]",
                 "Heat efficiency, annual average, net, radiators",
             ],
-            "2020": [1.0] * 7,
-            "source": ["source"] * 7,
-            "unit": ["unit"] * 7,
+            "2020": [1.0] * 8,
+            "source": ["source"] * 8,
+            "unit": ["unit"] * 8,
         }
     ).set_index(["technology", "parameter"])
 
     reference_output_df = pd.DataFrame(
         {
             "technology": [
+                "biogas upgrading",
                 "biogas upgrading",
                 "decentral gas boiler",
                 "decentral gas boiler connection",
                 "decentral gas boiler connection",
                 "heat pump",
             ],
             "parameter": [
+                "Technical lifetime",
                 "investment (upgrading, methane redution and grid injection)",
                 "Technical lifetime",
                 "Possible additional specific investment",
                 "Technical lifetime",
                 "Heat efficiency, annual average, net, radiators, existing one family house",
             ],
-            "2020": [1.0, 1.0, 1.0, 50.0, 1.0],
-            "source": ["source"] * 5,
-            "unit": ["unit"] * 5,
+            "2020": [1.0, 1.0, 1.0, 1.0, 50.0, 1.0],
+            "source": ["source"] * 6,
+            "unit": ["unit"] * 6,
         }
     )
     list_of_years = ["2020"]

outputs/US/costs_2020.csv

@@ -253,9 +253,11 @@ def test_pre_process_atb_input_file(config, input_file_year, year, expected):
     ]
     nrel_atb_technology_to_remove = config["nrel_atb"]["nrel_atb_technology_to_remove"]
     nrel_atb_source_link = config["nrel_atb"]["nrel_atb_source_link"]
+    nrel_atb_further_description = config["nrel_atb"]["nrel_atb_further_description"]
     output_df = pre_process_atb_input_file(
         input_file_path,
         nrel_atb_source_link,
+        nrel_atb_further_description,
         year,
         nrel_atb_columns_to_keep,
         nrel_atb_core_metric_parameter_to_keep,
@@ -379,13 +381,13 @@ def test_duplicate_fuel_cost(config):
 @pytest.mark.parametrize(
     "year, expected",
     [
-        (2020, (91, 7)),
-        (2025, (91, 7)),
-        (2030, (91, 7)),
-        (2035, (91, 7)),
-        (2040, (91, 7)),
-        (2045, (91, 7)),
-        (2050, (91, 7)),
+        (2020, (130, 9)),
+        (2025, (130, 9)),
+        (2030, (130, 9)),
+        (2035, (130, 9)),
+        (2040, (130, 9)),
+        (2045, (130, 9)),
+        (2050, (130, 9)),
     ],
 )
 def test_pre_process_manual_input_usa(config, year, expected):