Merge pull request #73 from Gromwud/main

Thesis update

Author: Gromwud

.dockerignore

@@ -0,0 +1,13 @@
+.git
+.gitignore
+.claude
+.idea
+projects/thesis/results
+projects/thesis/_test_archive
+__pycache__
+**/__pycache__
+*.pyc
+*.pdf
+*.rar
+Master_Thesis__Bavshin_.pdf
+moeadd.pdf

Dockerfile

@@ -0,0 +1,23 @@
+# python:3.10-slim has glibc, just enough to install the wheels.
+# numpy / scipy / sklearn / torch all ship manylinux wheels, so we
+# don't need a build toolchain at runtime.
+FROM python:3.10-slim
+
+# libgomp1 is needed by numpy/sklearn/torch for OpenMP threading.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /work
+
+# Install Python deps first so the layer caches across source edits.
+COPY requirements.txt /work/requirements.txt
+RUN pip install --no-cache-dir --upgrade pip \
+ && pip install --no-cache-dir -r requirements.txt
+
+# Copy the repo last so source edits don't bust the dep cache.
+COPY . /work
+
+# Default to interactive shell; the actual sweep command is supplied
+# by docker-compose via the per-service ``command:`` field.
+ENTRYPOINT ["/bin/bash", "-l"]

docker-compose.yml

@@ -0,0 +1,58 @@
+# 8 containers, one per ablation cell. All share the same image
+# and bind-mount projects/thesis/results/ so reps land on the host
+# filesystem and the aggregator can read them without entering a
+# container.
+#
+# Recommended invocation:
+#   docker compose build
+#   docker compose up -d              # runs all 8 in background
+#   docker compose logs -f cell-new   # tail one cell's output
+#   docker compose down               # stops everything
+#
+# Resource notes:
+#   - 16 host cores -> 2 threads/container is safe.
+#   - 32 host cores -> bump OMP/MKL/OPENBLAS_NUM_THREADS to 4 for
+#     ~half the wall-clock.
+#   - The runner skips finished <cell>_rep<NN>.json files on
+#     resume, so containers can be killed and relaunched freely.
+
+services:
+  cell-legacy: &cell-base
+    build: .
+    image: epde-thesis:latest
+    command: ["scripts/run_cell.sh", "legacy"]
+    volumes:
+      - ./projects/thesis/results:/work/projects/thesis/results
+    environment:
+      OMP_NUM_THREADS: "2"
+      MKL_NUM_THREADS: "2"
+      OPENBLAS_NUM_THREADS: "2"
+    restart: unless-stopped
+
+  cell-wape:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "wape"]
+
+  cell-instab:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "instab"]
+
+  cell-reg:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "reg"]
+
+  cell-wape-instab:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "wape_instab"]
+
+  cell-wape-reg:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "wape_reg"]
+
+  cell-instab-reg:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "instab_reg"]
+
+  cell-new:
+    <<: *cell-base
+    command: ["scripts/run_cell.sh", "new"]

epde/eq_mo_objectives.py

@@ -61,13 +61,20 @@ def equation_complexity_by_terms(system, equation_key):
 
 
 def _complexity_single_eq(system, equation_key):
+    # Index by ``weights_internal`` (always length ``len(structure)-1``,
+    # one entry per non-target term in structure order) rather than
+    # ``weights_final`` (zero-filtered to ``nnz+1`` by ``LASSOSparsity``
+    # and ``VWSRSparsity``): structure-position indexing breaks against
+    # ``weights_final`` whenever the sparsity step zeros more than one
+    # weight.
+    equation = system.vals[equation_key]
     eq_compl = 0
-    for idx, term in enumerate(system.vals[equation_key].structure):
-        if idx < system.vals[equation_key].target_idx:
-            if not system.vals[equation_key].weights_final[idx] == 0:
+    for idx, term in enumerate(equation.structure):
+        if idx < equation.target_idx:
+            if not equation.weights_internal[idx] == 0:
                 eq_compl += complexity_deriv(term.structure)
-        elif idx > system.vals[equation_key].target_idx:
-            if not system.vals[equation_key].weights_final[idx-1] == 0:
+        elif idx > equation.target_idx:
+            if not equation.weights_internal[idx-1] == 0:
                 eq_compl += complexity_deriv(term.structure)
         else:
             eq_compl += complexity_deriv(term.structure)

epde/interface/prepared_tokens.py

@@ -319,9 +319,22 @@ def __init__(self, labels = ['t',], max_power: int = 1, dimensionality=1):
 
         Args:
             dimensionality (`int`): optional, default - 1
-                data dimension 
+                data dimension
         """
-        assert len(labels) == dimensionality + 1, 'Incorrect labels for grids.'
+        # Two valid configurations:
+        # 1. One label per axis (legacy): ``['x_0', ..., 'x_N']`` with
+        #    ``len == dimensionality + 1``. Each axis is its own token,
+        #    plus the ``dim`` param redundantly encodes the same axis.
+        # 2. Single label (consolidated): ``['x']`` representing a single
+        #    family where the ``dim`` parameter is the ONLY axis
+        #    discriminator. Removes the redundant ``x_N`` prefix.
+        # ``set_status(unique_token_type=True)`` already restricts to one
+        # grid factor per term in either configuration, so behaviour is
+        # identical -- the consolidation is a labelling cleanup.
+        assert 1 <= len(labels) <= dimensionality + 1, (
+            f'Grid labels must be either a single name or one per axis; '
+            f'got {len(labels)} labels for dimensionality={dimensionality}'
+        )
 
         self._token_family = TokenFamily(token_type='grids')
         self._token_family.set_status(unique_specific_token=True, unique_token_type=True,

epde/operators/common/coeff_calculation.py

@@ -7,91 +7,104 @@
 """
 
 import numpy as np
-from sklearn.linear_model import LinearRegression, Ridge
+from sklearn.linear_model import LinearRegression
 
 import epde.globals as global_var
 from epde.operators.utils.template import CompoundOperator
 from epde.structure.main_structures import Equation
 
+
+# Marker attribute set by ``LASSOSparsity.apply`` on the equation
+# instance to indicate that the legacy LASSO post-processing refit
+# (LinearRegression on the LASSO survivors, un-normalised features)
+# should run on this equation. ``VWSRSparsity`` does NOT set this
+# marker; its PhysicsInformedLasso output is already on the physical
+# scale and would be corrupted by the refit. Gating on the equation
+# (rather than on the operator) keeps the strategy wiring untouched.
+LEGACY_REFIT_MARKER = '_legacy_refit_pending'
+
+
 class LinRegBasedCoeffsEquation(CompoundOperator):
     '''
-    
-    The operatror, dedicated to the calculation of the weights of the equation (for the free coefficient and 
-    each of its terms except the target one).
-
-    Attributes:
-        _tags (`set`): 
-        g_fun_vals (`numpy.ndarray`): 
-    
-    Methods:
-        apply(equation)
-            Calculate the coefficients of the equation, using the linear regression. The result is stored in the 
-            equation.weights_final attribute
+    Refit the LASSO survivors with ``LinearRegression`` on
+    *un-normalised* features, replacing ``weights_final`` with
+    physically-scaled coefficients.
+
+    Restores the legacy two-step pipeline:
+        1. LASSOSparsity fits Lasso on min-max-normalised features and
+           identifies the surviving (non-zero) terms.
+        2. This operator re-fits those survivors with ordinary least
+           squares on the un-normalised features to recover physical
+           coefficient magnitudes (LASSO coefficients are biased by
+           both L1 shrinkage and the upstream normalisation).
+
+    Gated by the per-equation marker ``LEGACY_REFIT_MARKER`` that
+    ``LASSOSparsity`` sets and ``VWSRSparsity`` leaves unset, so this
+    operator can be wired into both pipelines without a strategy flag.
+
+    Output shape matches the upstream sparsity convention:
+    ``np.append(coef_, intercept)`` -- one entry per surviving non-zero
+    feature plus a trailing intercept slot. Downstream consumers
+    (``L2Fitness.apply``, ``L2LRFitness.apply``) need no change.
     '''
     key = 'LinRegCoeffCalc'
-    
+
+    @staticmethod
+    def _legacy_evaluate_nonzero(objective: Equation):
+        """Build target + un-normalised feature matrix from the LASSO
+        survivors, independent of ``Equation.evaluate``.
+
+        Mirrors the pre-aaea0f4 legacy feature builder: iterate the
+        structure, skip the target, emit columns only for terms whose
+        ``weights_internal`` slot is non-zero. Returns
+        ``(target, features)`` with ``features=None`` when every
+        non-target slot was filtered to zero.
+        """
+        target = objective.structure[objective.target_idx].evaluate(False)
+        feats = []
+        for term_idx, term in enumerate(objective.structure):
+            if term_idx == objective.target_idx:
+                continue
+            wi_pos = (term_idx if term_idx < objective.target_idx
+                      else term_idx - 1)
+            if objective.weights_internal[wi_pos] != 0:
+                feats.append(term.evaluate(False))
+        if not feats:
+            return target, None
+        features = np.vstack(feats)
+        if features.ndim == 1:
+            features = np.expand_dims(features, 1).T
+        features = np.transpose(features)
+        return target, features
+
     def apply(self, objective : Equation, arguments : dict = None):
+        """Refit LASSO survivors with un-normalised LinearRegression.
+
+        Skipped unless ``LASSOSparsity`` set ``LEGACY_REFIT_MARKER`` on
+        the equation. The marker is cleared after the refit so a stale
+        marker from a previous run doesn't double-trigger work.
         """
-        Calculate the coefficients of the equation, using the linear regression.The result is stored in the 
-        objective.weights_final attribute
-
-        Args:
-            objective (`Equation`): the equation object, to that the fitness function is obtained.
-            arguments (`dict`): 
-            
-        Returns:
-            None
-        """        
-        # self_args, subop_args = self.parse_suboperator_args(arguments = arguments)
-        
-        assert objective.weights_internal_evald, 'Trying to calculate final weights before evaluating intermeidate ones (no sparsity).'
-        # target = objective.structure[objective.target_idx]
-        #
-        # target_vals = target.evaluate(False)
-        # features_vals = []
-        # nonzero_features_indexes = []
-        # for i in range(len(objective.structure)):
-        #     if i == objective.target_idx:
-        #         continue
-        #     idx = i if i < objective.target_idx else i-1
-        #     if objective.weights_internal[idx] != 0:
-        #         features_vals.append(objective.structure[i].evaluate(False))
-        #         nonzero_features_indexes.append(idx)
-        #
-        # if len(features_vals) == 0:
-        #     objective.weights_final = np.zeros(len(objective.structure))
-        # else:
-        #     features = features_vals[0]
-        #     if len(features_vals) > 1:
-        #         for i in range(1, len(features_vals)):
-        #             features = np.vstack([features, features_vals[i]])
-        #     features = np.vstack([features, np.ones(features_vals[0].shape)]) # Добавляем константную фичу
-        #     features = np.transpose(features)
-        #     estimator = LinearRegression(copy_X=True, fit_intercept=False, n_jobs=-1,
-        #                       positive=False, tol=0.0001)
-        #     # estimator = LinearRegression(fit_intercept=False)
-        #     if features.ndim == 1:
-        #         features = features.reshape(-1, 1)
-        #     try:
-        #         self.g_fun_vals = global_var.grid_cache.g_func[global_var.grid_cache.g_func != 0]
-        #     except AttributeError:
-        #         self.g_fun_vals = None
-        #     estimator.fit(features, target_vals, sample_weight = self.g_fun_vals)
-        #
-        #     valuable_weights = estimator.coef_
-        #     weights = np.zeros(len(objective.structure))
-        #     for weight_idx in range(len(weights)-1):
-        #         if weight_idx in nonzero_features_indexes:
-        #             weights[weight_idx] = valuable_weights[nonzero_features_indexes.index(weight_idx)]
-        #     weights[-1] = valuable_weights[-1]
-        #     objective.weights_final = weights
-        # _, target, features = objective.evaluate(normalize=False, return_val=False)
-        # self.g_fun_vals = global_var.grid_cache.g_func[global_var.grid_cache.g_func != 0]
-        # estimator = LinearRegression(copy_X=True, fit_intercept=True, n_jobs=-1, positive=False, tol=0.0001)
-        # estimator.fit(features, target, sample_weight=self.g_fun_vals)
-        # valuable_weights = estimator.coef_
-        # objective.weights_final = np.append(valuable_weights, estimator.intercept_)
-        # objective.weights_final_evald = True
-            
+        assert objective.weights_internal_evald, (
+            'Trying to calculate final weights before evaluating '
+            'intermediate ones (no sparsity).'
+        )
+        if not getattr(objective, LEGACY_REFIT_MARKER, False):
+            return
+
+        target, features = self._legacy_evaluate_nonzero(objective)
+        if features is None:
+            # No non-zero terms to refit -- leave ``weights_final`` as
+            # set by the upstream sparsity step (just the intercept).
+            objective.weights_final_evald = True
+            setattr(objective, LEGACY_REFIT_MARKER, False)
+            return
+
+        self.g_fun_vals = global_var.grid_cache.g_func[global_var.grid_cache.g_func_mask]
+        estimator = LinearRegression(copy_X=True, fit_intercept=True, n_jobs=-1)
+        estimator.fit(features, target, sample_weight=self.g_fun_vals)
+        objective.weights_final = np.append(estimator.coef_, estimator.intercept_)
+        objective.weights_final_evald = True
+        setattr(objective, LEGACY_REFIT_MARKER, False)
+
     def use_default_tags(self):
         self._tags = {'coefficient calculation', 'gene level', 'no suboperators', 'inplace'}