Pierresegonne/remove production per unit (#8119)

pierresegonne · VIKTORVAV99 · web-flow · commit 26a49357510c · 2025-05-06T13:10:50.000Z
* feat: add enum for parser data types

* clean up parser data types throughout contrib

* chore: remove references to productionPerUnit

* remove unused code

---------

Co-authored-by: Pierre Segonne &lt;pierre.segonne@electricitymaps.com&gt;
Co-authored-by: Viktor Andersson &lt;30777521+VIKTORVAV99@users.noreply.github.com&gt;
diff --git a/config/zones/DK-DK1.yaml b/config/zones/DK-DK1.yaml
@@ -518,7 +518,6 @@ parsers:
   production: ENTSOE.fetch_production
   productionCapacity: ENTSOE.fetch_production_capacity
   productionPerModeForecast: DK.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/DK-DK2.yaml b/config/zones/DK-DK2.yaml
@@ -503,7 +503,6 @@ parsers:
   production: ENTSOE.fetch_production
   productionCapacity: ENTSOE.fetch_production_capacity
   productionPerModeForecast: DK.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/FI.yaml b/config/zones/FI.yaml
@@ -439,7 +439,6 @@ parsers:
   production: ENTSOE.fetch_production
   productionCapacity: FI.fetch_production_capacity
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/PL.yaml b/config/zones/PL.yaml
@@ -576,7 +576,6 @@ parsers:
   production: ENTSOE.fetch_production
   productionCapacity: ENTSOE.fetch_production_capacity
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/SE-SE1.yaml b/config/zones/SE-SE1.yaml
@@ -293,7 +293,6 @@ parsers:
   price: NORDPOOL.fetch_price
   production: ENTSOE.fetch_production
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/SE-SE2.yaml b/config/zones/SE-SE2.yaml
@@ -379,7 +379,6 @@ parsers:
   price: NORDPOOL.fetch_price
   production: ENTSOE.fetch_production
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/SE-SE3.yaml b/config/zones/SE-SE3.yaml
@@ -379,7 +379,6 @@ parsers:
   price: NORDPOOL.fetch_price
   production: ENTSOE.fetch_production
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/SE-SE4.yaml b/config/zones/SE-SE4.yaml
@@ -364,7 +364,6 @@ parsers:
   price: NORDPOOL.fetch_price
   production: ENTSOE.fetch_production
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 sources:
   EU-ETS, ENTSO-E 2021:
diff --git a/config/zones/XK.yaml b/config/zones/XK.yaml
@@ -214,6 +214,5 @@ parsers:
   production: ENTSOE.fetch_production
   productionCapacity: ENTSOE.fetch_production_capacity
   productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts
-  productionPerUnit: ENTSOE.fetch_production_per_units
 region: Europe
 timezone: Europe/Belgrade
diff --git a/electricitymap/contrib/config/model.py b/electricitymap/contrib/config/model.py
@@ -102,7 +102,6 @@ class Parsers(ParsersBaseModel):
     realtimeLocationalMarginalPrice: str | None
     price: str | None
     production: str | None
-    productionPerUnit: str | None
     productionCapacity: str | None
 
 
@@ -117,7 +116,6 @@ class Delays(StrictBaseModel):
     price: PositiveInt | None
     production: PositiveInt | None
     productionPerModeForecast: PositiveInt | None
-    productionPerUnit: PositiveInt | None
 
 
 class Zone(StrictBaseModelWithAlias):
diff --git a/electricitymap/contrib/lib/data_types.py b/electricitymap/contrib/lib/data_types.py
@@ -12,8 +12,6 @@ class ParserDataType(Enum):
     PRODUCTION = "production"
     PRODUCTION_PER_MODE_FORECAST = "productionPerModeForecast"
     REALTIME_LOCATIONAL_MARGINAL_PRICE = "realtimeLocationalMarginalPrice"
-    # TODO: Deprecate
-    PRODUCTION_PER_UNIT = "productionPerUnit"
     # TODO: Double check if we should keep them here?
     PRODUCTION_CAPACITY = "productionCapacity"
 
diff --git a/parsers/ENTSOE.py b/parsers/ENTSOE.py
@@ -202,19 +202,6 @@ def __str__(self) -> str:
     "XK": "10Y1001C--00100H",
 }
 
-# Generation per unit can only be obtained at EIC (Control Area) level
-ENTSOE_EIC_MAPPING: dict[str, str] = {
-    "DK-DK1": "10Y1001A1001A796",
-    "DK-DK2": "10Y1001A1001A796",
-    "FI": "10YFI-1--------U",
-    "PL": "10YPL-AREA-----S",
-    "SE-SE1": "10YSE-1--------K",
-    "SE-SE2": "10YSE-1--------K",
-    "SE-SE3": "10YSE-1--------K",
-    "SE-SE4": "10YSE-1--------K",
-    # TODO: ADD DE
-}
-
 # Define zone_keys to an array of zone_keys for aggregated production data
 ZONE_KEY_AGGREGATES: dict[str, list[str]] = {
     "IT-SO": ["IT-CA", "IT-SO"],
@@ -270,103 +257,6 @@ def __str__(self) -> str:
     "LU": ENTSOE_DOMAIN_MAPPINGS["DE-LU"],
 }
 
-ENTSOE_UNITS_TO_ZONE: dict[str, str] = {
-    # DK-DK1
-    "Anholt": "DK-DK1",
-    "Esbjergvaerket 3": "DK-DK1",
-    "Fynsvaerket 7": "DK-DK1",
-    "Horns Rev A": "DK-DK1",
-    "Horns Rev B": "DK-DK1",
-    "Nordjyllandsvaerket 3": "DK-DK1",
-    "Silkeborgvaerket": "DK-DK1",
-    "Skaerbaekvaerket 3": "DK-DK1",
-    "Studstrupvaerket 3": "DK-DK1",
-    "Studstrupvaerket 4": "DK-DK1",
-    # DK-DK2
-    "Amagervaerket 3": "DK-DK2",
-    "Asnaesvaerket 2": "DK-DK2",
-    "Asnaesvaerket 5": "DK-DK2",
-    "Avedoerevaerket 1": "DK-DK2",
-    "Avedoerevaerket 2": "DK-DK2",
-    "Kyndbyvaerket 21": "DK-DK2",
-    "Kyndbyvaerket 22": "DK-DK2",
-    "Roedsand 1": "DK-DK2",
-    "Roedsand 2": "DK-DK2",
-    # FI
-    "Alholmens B2": "FI",
-    "Haapavesi B1": "FI",
-    "Kaukaan Voima G10": "FI",
-    "Keljonlahti B1": "FI",
-    "Loviisa 1 G11": "FI",
-    "Loviisa 1 G12": "FI",
-    "Loviisa 2 G21": "FI",
-    "Loviisa 2 G22": "FI",
-    "Olkiluoto 1 B1": "FI",
-    "Olkiluoto 2 B2": "FI",
-    "Toppila B2": "FI",
-    # SE-SE1
-    "Bastusel G1": "SE-SE1",
-    "Gallejaur G1": "SE-SE1",
-    "Gallejaur G2": "SE-SE1",
-    "Harsprånget G1": "SE-SE1",
-    "Harsprånget G2": "SE-SE1",
-    "Harsprånget G4": "SE-SE1",
-    "Harsprånget G5": "SE-SE1",
-    "Letsi G1": "SE-SE1",
-    "Letsi G2": "SE-SE1",
-    "Letsi G3": "SE-SE1",
-    "Ligga G3": "SE-SE1",
-    "Messaure G1": "SE-SE1",
-    "Messaure G2": "SE-SE1",
-    "Messaure G3": "SE-SE1",
-    "Porjus G11": "SE-SE1",
-    "Porjus G12": "SE-SE1",
-    "Porsi G3": "SE-SE1",
-    "Ritsem G1": "SE-SE1",
-    "Seitevare G1": "SE-SE1",
-    "Vietas G1": "SE-SE1",
-    "Vietas G2": "SE-SE1",
-    # SE-SE2
-    "Stalon G1": "SE-SE2",
-    "Stornorrfors G1": "SE-SE2",
-    "Stornorrfors G2": "SE-SE2",
-    "Stornorrfors G3": "SE-SE2",
-    "Stornorrfors G4": "SE-SE2",
-    # SE-SE3
-    "Forsmark block 1 G11": "SE-SE3",
-    "Forsmark block 1 G12": "SE-SE3",
-    "Forsmark block 2 G21": "SE-SE3",
-    "Forsmark block 2 G22": "SE-SE3",
-    "Forsmark block 3 G31": "SE-SE3",
-    "KVV Västerås G3": "SE-SE3",
-    "KVV1 Värtaverket": "SE-SE3",
-    "KVV6 Värtaverket": "SE-SE3",
-    "KVV8 Värtaverket": "SE-SE3",
-    "Oskarshamn G3": "SE-SE3",
-    "Oskarshamn G1Ö+G1V": "SE-SE3",
-    "Ringhals block 1 G11": "SE-SE3",
-    "Ringhals block 1 G12": "SE-SE3",
-    "Ringhals block 2 G21": "SE-SE3",
-    "Ringhals block 2 G22": "SE-SE3",
-    "Ringhals block 3 G31": "SE-SE3",
-    "Ringhals block 3 G32": "SE-SE3",
-    "Ringhals block 4 G41": "SE-SE3",
-    "Ringhals block 4 G42": "SE-SE3",
-    "Rya KVV": "SE-SE3",
-    "Stenungsund B3": "SE-SE3",
-    "Stenungsund B4": "SE-SE3",
-    "Trängslet G1": "SE-SE3",
-    "Trängslet G2": "SE-SE3",
-    "Trängslet G3": "SE-SE3",
-    "Uppsala KVV": "SE-SE3",
-    "Åbyverket Örebro": "SE-SE3",
-    # SE-SE4
-    "Gasturbiner Halmstad G12": "SE-SE4",
-    "Karlshamn G1": "SE-SE4",
-    "Karlshamn G2": "SE-SE4",
-    "Karlshamn G3": "SE-SE4",
-}
-
 VALIDATIONS: dict[str, dict[str, Any]] = {
     # This is a list of criteria to ensure validity of data,
     # used in validate_production()
@@ -547,32 +437,6 @@ def query_production(
     )
 
 
-def query_production_per_units(
-    psr_type: str,
-    domain: str,
-    session: Session,
-    target_datetime: datetime | None = None,
-) -> str | None:
-    params = {
-        # Actual generation - A document providing the actual generation for a
-        # period.
-        "documentType": "A73",
-        # Realised - The process for the treatment of realised data as opposed
-        # to forecast data
-        "processType": "A16",
-        "psrType": psr_type,
-        "in_Domain": domain,
-    }
-    # Note: ENTSOE only supports 1d queries for this type
-    return query_ENTSOE(
-        session,
-        params,
-        target_datetime=target_datetime,
-        span=(-24, 0),
-        function_name=query_production_per_units.__name__,
-    )
-
-
 def query_exchange(
     in_domain: str,
     out_domain: str,
@@ -920,56 +784,6 @@ def _group_production_data_by_datetime(
     return grouped_data
 
 
-def parse_production_per_units(xml_text: str) -> Any | None:
-    values = {}
-
-    if not xml_text:
-        return None
-    soup = BeautifulSoup(xml_text, "html.parser")
-    # Get all points
-    for timeseries in soup.find_all("timeseries"):
-        resolution = str(timeseries.find("resolution").contents[0])
-        datetime_start = datetime.fromisoformat(
-            zulu_to_utc(timeseries.find("start").contents[0])
-        )
-        is_production = bool(timeseries.find("inBiddingZone_Domain.mRID".lower()))
-        psr_type = str(timeseries.find("mktpsrtype").find("psrtype").contents[0])
-        unit_key = str(
-            timeseries.find("mktpsrtype")
-            .find("powersystemresources")
-            .find("mrid")
-            .contents[0]
-        )
-        unit_name = str(
-            timeseries.find("mktpsrtype")
-            .find("powersystemresources")
-            .find("name")
-            .contents[0]
-        )
-        if not is_production:
-            continue
-        for entry in timeseries.find_all("point"):
-            quantity = float(entry.find("quantity").contents[0])
-            position = int(entry.find("position").contents[0])
-            dt = datetime_from_position(datetime_start, position, resolution)
-            key = (unit_key, dt)
-            if key in values:
-                if is_production:
-                    values[key]["production"] += quantity
-                else:
-                    values[key]["production"] -= quantity
-            else:
-                values[key] = {
-                    "datetime": dt,
-                    "production": quantity,
-                    "productionType": ENTSOE_PARAMETER_BY_GROUP[psr_type],
-                    "unitKey": unit_key,
-                    "unitName": unit_name,
-                }
-
-    return values.values()
-
-
 def parse_exchange(
     xml_text: str,
     is_import: bool,
@@ -1134,57 +948,6 @@ def fetch_production(
     return [x for x in aggregated_zone_data if validate_production(x, logger)]
 
 
-@refetch_frequency(timedelta(days=1))
-def fetch_production_per_units(
-    zone_key: str,
-    session: Session = Session(),
-    target_datetime: datetime | None = None,
-    logger: Logger = getLogger(__name__),
-) -> list:
-    """Returns all production units and production values."""
-
-    # If no target_datetime is specified, or the target datetime is less
-    # than 5 days ago we set the target_datetime to 5 days ago.
-    if target_datetime is None or target_datetime > datetime.now(
-        tz=timezone.utc
-    ) - timedelta(days=5):
-        logger.info(
-            "This dataset has a publishing guideline of 5 days from the current MTU, setting the target_datetime to 5 days ago to get the latest data."
-        )
-        target_datetime = datetime.now(tz=timezone.utc) - timedelta(days=5)
-
-    domain = ENTSOE_EIC_MAPPING[zone_key]
-    data = []
-    # Iterate over all psr types
-    for k in ENTSOE_PARAMETER_DESC:
-        try:
-            raw_production_per_units = query_production_per_units(
-                k, domain, session, target_datetime
-            )
-            if raw_production_per_units is not None:
-                values = parse_production_per_units(raw_production_per_units) or []
-                for v in values:
-                    if not v:
-                        continue
-                    v["source"] = "entsoe.eu"
-                    if v["unitName"] not in ENTSOE_UNITS_TO_ZONE:
-                        logger.warning(
-                            f"Unknown unit {v['unitName']} with id {v['unitKey']}"
-                        )
-                    else:
-                        v["zoneKey"] = ENTSOE_UNITS_TO_ZONE[v["unitName"]]
-                        if v["zoneKey"] == zone_key:
-                            data.append(v)
-        except Exception as e:
-            raise ParserException(
-                parser="ENTSOE.py",
-                message=f"Failed to fetch data for {k} in {zone_key}",
-                zone_key=zone_key,
-            ) from e
-
-    return data
-
-
 def get_raw_exchange(
     zone_key1: ZoneKey,
     zone_key2: ZoneKey,
diff --git a/parsers/lib/parsers.py b/parsers/lib/parsers.py
@@ -30,9 +30,6 @@
     ParserDataType.PRODUCTION_PER_MODE_FORECAST: PRODUCTION_PER_MODE_FORECAST_PARSERS,
     ParserDataType.REALTIME_LOCATIONAL_MARGINAL_PRICE: REALTIME_LOCATIONAL_MARGINAL_PRICE_PARSERS,
     ParserDataType.PRODUCTION_CAPACITY: PRODUCTION_CAPACITY_PARSERS,
-    # TODO - support productionPerUnit for now, remove support in a follow-up PR
-    # https://electricitymaps.slack.com/archives/C08LH4L9PFB/p1746519910179509
-    ParserDataType.PRODUCTION_PER_UNIT: PRODUCTION_PER_UNIT_PARSERS,
 }
 
 
diff --git a/tests/test_parser_interface.py b/tests/test_parser_interface.py
@@ -30,7 +30,6 @@
     "price": _PARSER_FUNCTION_ARGS,
     "production": _PARSER_FUNCTION_ARGS,
     "productionPerModeForecast": _PARSER_FUNCTION_ARGS,
-    "productionPerUnit": _PARSER_FUNCTION_ARGS,
     "productionCapacity": _CAPACITY_PARSER_FUNCTION_ARGS,
 }
 _RETURN_PARSER_TYPE = [
@@ -55,7 +54,6 @@
     "price": _RETURN_PARSER_TYPE,
     "production": _RETURN_PARSER_TYPE,
     "productionPerModeForecast": _RETURN_PARSER_TYPE,
-    "productionPerUnit": _RETURN_PARSER_TYPE,
     "productionCapacity": _RETURN_PARSER_TYPE,
 }
 
diff --git a/web/geo/types.ts b/web/geo/types.ts
