coreneuron: test reports (mostly compartment and summation) (#337)

## Context

This should finally take care of the compartment sets for coreneuron

Fix: #345 #322

testing for:

neuronsimulator/nrn#3507
neuronsimulator/nrn#3542

## Scope

- [x] rework report handling (mostly in node and target_manager this time. i.e. removal of `_report_setup`)
- [x] new `CoreReportConfig` to finally handle in a proper way `report.conf`
- [x] remove `write_sim_config`, `update_report_conf`, `write_report_conf` and similar functions in `CoreConfig`. Use `CoreSimulationConfig` and `CoreReportConfig` instead
- [x] move `cell-permute` in another issue/pr. EDIT done: #367
- [x] scaling moved to `report_parameters.py`
- [x] `CumulativeError` moved in `pyutils`
- [x] `ReportParams` now has its own file alleviating the burden of the `node.py` file. `_report_build_params` (previously in `node.py`) is also inside the new file.
- [x] `target_type` is no more. We do not need encoding/decoding. We just write `sections` and `compartments` in the `report.conf` 
- [x] streamlined `enable_reports`. It is still a complex function but it should be easier to read and with less edge cases 
- [x] use the decorator `@cache_errors` to just fill a `CumulativeError` instead of raising an error and raise it at the end
- [x] `merge_dicts` (in `tests/utils.py`) allows for overrides and suppression of sections from the child dict with the special keywords: `override_field` and `delete_field`

### `report.conf` changes

- [x] `type_name` has been split into `sections` and `compartments`. No need to encode everything in an enum to decode later. This simplifies code and makes the `report.conf`. The code is slightly slower `0(1)` in a part of the code that was not performance-critical
- [x] `scaling` added at the end of the report line (after buffer size)
- [x] add 2 additional binary lines in case of a compartment set report to pass the exact positions of the compartment sets. They are necessary if and only if the report type is `compartment_set`. Having them when it is not or not having them when it is is an error.

## Testing

- [x] add integration-e2e tests in `test_reports.py`

## Testing
- [x] test various bug fixes in coreneuron
- [x] add lfp test. Check that it fails with neuron
- [x] fix integration-e2e
- [x] fix scientific
- [x] fix unit

Author: cattabiani

.github/workflows/simulation_test.yml

@@ -33,7 +33,7 @@ on:
         required: false
 
 env:
-  NEURON_COMMIT_ID: '7175203'
+  NEURON_COMMIT_ID: '0d990513b'
   RDMAV_FORK_SAFE: '1'
 
 jobs:
@@ -159,6 +159,7 @@ jobs:
         path: nrn
         key: ${{ matrix.os }}-neuron-${{ env.NEURON_BRANCH }}-${{ env.NEURON_COMMIT_ID }}-py${{ matrix.python-version }}
 
+
     - name: Install NEURON
       if: steps.cache-neuron.outputs.cache-hit != 'true'
       run: |

.gitignore

@@ -33,5 +33,4 @@ dist/*
 sdist/*
 docs/build/
 cover/*
-MANIFEST
-removeme
+MANIFEST

neurodamus/core/coreneuron_configuration.py

@@ -1,10 +1,7 @@
-import logging
-import os
 from pathlib import Path
 
 from . import NeuronWrapper as Nd
-from ._utils import run_only_rank0
-from .configuration import ConfigurationError, SimConfig
+from .configuration import SimConfig
 
 
 class CompartmentMapping:
@@ -127,186 +124,6 @@ def restore_path(self):
     def instantiate_artificial_cell(self):
         self.artificial_cell_object = Nd.CoreNEURONArtificialCell()
 
-    @run_only_rank0
-    def update_report_config(self, substitutions):
-        """Updates a report configuration (e.g., stop time).
-
-        Searches for the specified report and nodeset, updates the relevant parameters
-        (currently only `tstop`), and writes the updated configuration to a new file.
-
-        Note: `report.conf` must already exist.
-        """
-        report_conf = Path(self.report_config_file_save)
-
-        # Read all content
-        with report_conf.open("rb") as f:
-            lines = f.readlines()
-
-        # Track performed substitutions
-        applied_subs = set()
-
-        # Find and update the matching line
-        for i, line in enumerate(lines):
-            try:
-                parts = line.decode().split()
-                key = tuple(parts[0:2])  # Report name and target name
-
-                if key in substitutions:
-                    # This is often but not always tstop:
-                    # new_tend = min(tstop, tend) where tend is the ending
-                    # of the report and tstop is the tstop of this simulation
-                    # (potentially between a restore and a save)
-                    new_tend = substitutions[key]
-                    parts[9] = f"{new_tend:.6f}"
-                    lines[i] = (" ".join(parts) + "\n").encode()
-                    applied_subs.add(key)
-            except (UnicodeDecodeError, IndexError):  # noqa: PERF203
-                # Ignore lines that cannot be decoded (binary data)
-                continue
-
-        # Find substitutions that were not applied
-        missing_subs = set(substitutions.keys()) - applied_subs
-
-        if missing_subs:
-            raise ConfigurationError(
-                f"Some substitutions could not be applied for the following "
-                f"(report, target) pairs: {missing_subs}"
-            )
-
-        with report_conf.open("wb") as f:
-            f.writelines(lines)
-
-    @run_only_rank0
-    def write_report_config(
-        self,
-        report_name,
-        target_name,
-        report_type,
-        report_variable,
-        unit,
-        report_format,
-        target_type,
-        dt,
-        start_time,
-        end_time,
-        gids,
-        buffer_size=8,
-    ):
-        """Here we append just one report entry to report.conf. We are not writing the full file as
-        this is done incrementally in Node.enable_reports
-        """
-        import struct
-
-        num_gids = len(gids)
-        logging.info("Adding report %s for CoreNEURON with %s gids", report_name, num_gids)
-        report_conf = Path(self.report_config_file_save)
-        report_conf.parent.mkdir(parents=True, exist_ok=True)
-        with report_conf.open("ab") as fp:
-            # Write the formatted string to the file
-            fp.write(
-                (
-                    "%s %s %s %s %s %s %d %lf %lf %lf %d %d\n"  # noqa: UP031
-                    % (
-                        report_name,
-                        target_name,
-                        report_type,
-                        report_variable,
-                        unit,
-                        report_format,
-                        target_type,
-                        dt,
-                        start_time,
-                        end_time,
-                        num_gids,
-                        buffer_size,
-                    )
-                ).encode()
-            )
-            # Write the array of integers to the file in binary format
-            fp.write(struct.pack(f"{num_gids}i", *gids))
-            fp.write(b"\n")
-
-    @run_only_rank0
-    def write_sim_config(
-        self,
-        tstop: float,
-        dt: float,
-        prcellgid: int,
-        celsius: float,
-        v_init: float,
-        pattern=None,
-        seed=None,
-        model_stats=False,
-        enable_reports=True,
-    ):
-        """Writes the simulation configuration to a file.
-
-        Args:
-            tstop (float): Simulation stop time.
-            dt (float): Time step for the simulation.
-            prcellgid (int): dump cell state GID. CoreNeuron allows only one
-                cell to be dumped at a time.
-            celsius (float): Temperature in Celsius.
-            v_init (float): Initial voltage.
-            pattern (str, optional): Pattern for the simulation. Defaults to None.
-            seed (int, optional): Random seed for the simulation. Defaults to None.
-            model_stats (bool, optional): Flag to enable model statistics. Defaults to False.
-            enable_reports (bool, optional): Flag to enable reports. Defaults to True.
-        """
-        simconf = Path(self.sim_config_file)
-        logging.info("Writing sim config file: %s", simconf)
-        simconf.parent.mkdir(parents=True, exist_ok=True)
-
-        with simconf.open("w", encoding="utf-8") as fp:
-            fp.write(f"outpath='{os.path.abspath(self.output_root)}'\n")
-            fp.write(f"datpath='{os.path.abspath(self.datadir)}'\n")
-            fp.write(f"tstop={tstop}\n")
-            fp.write(f"dt={dt}\n")
-            fp.write(f"prcellgid={prcellgid}\n")
-            fp.write(f"celsius={celsius}\n")
-            fp.write(f"voltage={v_init}\n")
-            fp.write(f"cell-permute={int(self.default_cell_permute)}\n")
-            if pattern:
-                fp.write(f"pattern='{pattern}'\n")
-            if seed:
-                fp.write(f"seed={int(seed)}\n")
-            if model_stats:
-                fp.write("'model-stats'\n")
-            if enable_reports:
-                fp.write(f"report-conf='{self.report_config_file_save}'\n")
-            fp.write(f"mpi={os.environ.get('NEURON_INIT_MPI', '1')}\n")
-
-        logging.info(" => Dataset written to '%s'", simconf)
-
-    @run_only_rank0
-    def write_report_count(self, count, mode="w"):
-        report_config = Path(self.report_config_file_save)
-        report_config.parent.mkdir(parents=True, exist_ok=True)
-        with report_config.open(mode) as fp:
-            fp.write(f"{count}\n")
-
-    @run_only_rank0
-    def write_population_count(self, count):
-        self.write_report_count(count, mode="a")
-
-    @run_only_rank0
-    def write_spike_population(self, population_name, population_offset=None):
-        report_config = Path(self.report_config_file_save)
-        report_config.parent.mkdir(parents=True, exist_ok=True)
-        with report_config.open("a", encoding="utf-8") as fp:
-            fp.write(population_name)
-            if population_offset is not None:
-                fp.write(f" {int(population_offset)}")
-            fp.write("\n")
-
-    @run_only_rank0
-    def write_spike_filename(self, filename):
-        report_config = Path(self.report_config_file_save)
-        report_config.parent.mkdir(parents=True, exist_ok=True)
-        with report_config.open("a", encoding="utf-8") as fp:
-            fp.write(filename)
-            fp.write("\n")
-
     def psolve_core(self, coreneuron_direct_mode=False):
         from neuron import coreneuron