From ba5f572b4b250a8e127a7ed82bff6abc00ccdbc1 Mon Sep 17 00:00:00 2001 From: makiper Date: Tue, 11 Nov 2025 09:21:49 -0800 Subject: [PATCH 1/3] PR for edge correction --- src/cover_class/config/static.yml | 4 +++- src/cover_class/static/preprocessor.py | 16 +++++++++++++++- src/cover_class/static/retrieval.py | 9 +++++++-- tests/static/retrieval_test.py | 23 +++++++++++++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/cover_class/config/static.yml b/src/cover_class/config/static.yml index f23fba1..dd4fad2 100644 --- a/src/cover_class/config/static.yml +++ b/src/cover_class/config/static.yml @@ -21,4 +21,6 @@ datasets: - /some/path water: - http:example.com - - /some/path \ No newline at end of file + - /some/path +left-edge-correction: + - /some/path/to/hdf5: [100, 115, 200] \ No newline at end of file diff --git a/src/cover_class/static/preprocessor.py b/src/cover_class/static/preprocessor.py index 8e3aaa9..c0994fa 100644 --- a/src/cover_class/static/preprocessor.py +++ b/src/cover_class/static/preprocessor.py @@ -1,4 +1,4 @@ -from typing import Tuple +from typing import Tuple, List from torch import FloatTensor, Tensor import torch from numpy.typing import NDArray @@ -19,3 +19,17 @@ def interior_interpolation( def convolve(data_matrix:FloatTensor) -> FloatTensor: ... # type: ignore +def left_edge_scale( + data_matrix: NDArray[np.float32], + left_edges: List[int] + ) -> None: + """ + This function takes in the data matrix and the left edges of the edge discontinuity. + It then scales the left side of the spectra to be on the same magnitude. + As such, there will be cumulaive scaling of the left-portion of the data matrix until the + edges are all sequentially processed/corrected. + """ + for edge in left_edges: + denom = np.where(data_matrix[:, edge] == 0, 1e-8, data_matrix[:, edge]) # division by 0 protection + scaling_factors = data_matrix[:, edge+1] / denom + data_matrix[:, :edge+1] *= scaling_factors[:, np.newaxis] diff --git a/src/cover_class/static/retrieval.py b/src/cover_class/static/retrieval.py index ce59b03..e06ccff 100644 --- a/src/cover_class/static/retrieval.py +++ b/src/cover_class/static/retrieval.py @@ -9,7 +9,7 @@ import requests # type: ignore[import] from cover_class.utils import read_config -from cover_class.static.preprocessor import interior_interpolation +from cover_class.static.preprocessor import interior_interpolation, left_edge_scale def download(uri: str) -> Tuple[NDArray[np.float32], NDArray[np.float32]]: @@ -85,14 +85,19 @@ def generate_hdf5_from_config(config_path:str) -> None: ds = config['datasets'] outdir = ds['output-directory'] assert Path(outdir).is_dir(), f"'output-directory': {outdir} is not a directory" + edges = config.get('left-edge-correction', dict({})) for d in (ds_classes := ds['classes']): if ds_classes[d] == None: continue for location in ds_classes[d]: - # 1. get the wavelength and spectra from the locations + # 1a. get the wavelength and spectra from the locations if Path(location).is_file(): file_wavelengths, spectra = vfs_csv(location) else: file_wavelengths, spectra = download(location) + # [Optional] 1b. correct for the left edges + if d in edges: + left_edge_scale(spectra, edges[d]) + # 2. interpolate the wavelengths spectra = spectra[~np.isnan(spectra).any(axis=1)] spectra_interp, target_wavelengths = interior_interpolation(spectra, file_wavelengths) diff --git a/tests/static/retrieval_test.py b/tests/static/retrieval_test.py index 6b31a95..7df1344 100644 --- a/tests/static/retrieval_test.py +++ b/tests/static/retrieval_test.py @@ -6,6 +6,7 @@ import torch import io import contextlib +from copy import deepcopy from cover_class.static import retrieval # type: ignore[import] MODULE = "cover_class.static.retrieval" @@ -95,6 +96,28 @@ def test_vfs_csv(self): np.testing.assert_allclose(wl, wls) np.testing.assert_allclose(sp, spectra) + def test_left_edge_correction(self): + data = np.array([ + [1.0, 2.0, 4.0, 8.0], + [2.0, 4.0, 8.0, 16.0], + ], dtype=np.float32) + d0 = deepcopy(data) + + left_edges = [1] + expected = np.array([ + [2.0, 4.0, 4.0, 8.0], + [4.0, 8.0, 8.0, 16.0], + ], dtype=np.float32) + retrieval.left_edge_scale(d0, left_edges) + np.testing.assert_allclose(d0, expected, rtol=1e-6) + + left_edges = [1,2] + expected = np.array([ + [4.0, 8.0, 8.0, 8.0], + [8.0, 16.0, 16.0, 16.0], + ], dtype=np.float32) + retrieval.left_edge_scale(data, left_edges) + np.testing.assert_allclose(data, expected, rtol=1e-6) if __name__ == "__main__": unittest.main() From 5760ad5dfb6ef7f317180eb1a3cfd3a697d33290 Mon Sep 17 00:00:00 2001 From: makiper Date: Tue, 11 Nov 2025 09:29:18 -0800 Subject: [PATCH 2/3] made the function robust to out of order edge indices --- src/cover_class/static/preprocessor.py | 2 +- tests/static/retrieval_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cover_class/static/preprocessor.py b/src/cover_class/static/preprocessor.py index c0994fa..1e48743 100644 --- a/src/cover_class/static/preprocessor.py +++ b/src/cover_class/static/preprocessor.py @@ -29,7 +29,7 @@ def left_edge_scale( As such, there will be cumulaive scaling of the left-portion of the data matrix until the edges are all sequentially processed/corrected. """ - for edge in left_edges: + for edge in sorted(left_edges): denom = np.where(data_matrix[:, edge] == 0, 1e-8, data_matrix[:, edge]) # division by 0 protection scaling_factors = data_matrix[:, edge+1] / denom data_matrix[:, :edge+1] *= scaling_factors[:, np.newaxis] diff --git a/tests/static/retrieval_test.py b/tests/static/retrieval_test.py index 7df1344..1e56807 100644 --- a/tests/static/retrieval_test.py +++ b/tests/static/retrieval_test.py @@ -102,6 +102,7 @@ def test_left_edge_correction(self): [2.0, 4.0, 8.0, 16.0], ], dtype=np.float32) d0 = deepcopy(data) + d1 = deepcopy(data) left_edges = [1] expected = np.array([ @@ -116,6 +117,11 @@ def test_left_edge_correction(self): [4.0, 8.0, 8.0, 8.0], [8.0, 16.0, 16.0, 16.0], ], dtype=np.float32) + retrieval.left_edge_scale(d1, left_edges) + np.testing.assert_allclose(d1, expected, rtol=1e-6) + + # test out of order indices + left_edges = [2, 1] retrieval.left_edge_scale(data, left_edges) np.testing.assert_allclose(data, expected, rtol=1e-6) From b350bcd03f9afc022bdced4d65897c9ff0356af8 Mon Sep 17 00:00:00 2001 From: makiper Date: Tue, 11 Nov 2025 09:55:31 -0800 Subject: [PATCH 3/3] fixed typos --- src/cover_class/static/preprocessor.py | 2 +- src/cover_class/static/retrieval.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cover_class/static/preprocessor.py b/src/cover_class/static/preprocessor.py index 1e48743..c0847f7 100644 --- a/src/cover_class/static/preprocessor.py +++ b/src/cover_class/static/preprocessor.py @@ -26,7 +26,7 @@ def left_edge_scale( """ This function takes in the data matrix and the left edges of the edge discontinuity. It then scales the left side of the spectra to be on the same magnitude. - As such, there will be cumulaive scaling of the left-portion of the data matrix until the + As such, there will be cumulative scaling of the left-portion of the data matrix until the edges are all sequentially processed/corrected. """ for edge in sorted(left_edges): diff --git a/src/cover_class/static/retrieval.py b/src/cover_class/static/retrieval.py index e06ccff..f58db88 100644 --- a/src/cover_class/static/retrieval.py +++ b/src/cover_class/static/retrieval.py @@ -95,8 +95,8 @@ def generate_hdf5_from_config(config_path:str) -> None: else: file_wavelengths, spectra = download(location) # [Optional] 1b. correct for the left edges - if d in edges: - left_edge_scale(spectra, edges[d]) + if location in edges: + left_edge_scale(spectra, edges[location]) # 2. interpolate the wavelengths spectra = spectra[~np.isnan(spectra).any(axis=1)]