From 5150833cdc6673c5f2e1acffda6a1ac2494d54de Mon Sep 17 00:00:00 2001 From: Juan Marulanda Date: Tue, 12 Jul 2022 18:41:52 -0400 Subject: [PATCH 1/3] fixed empty node for normalized heald reader --- .pre-commit-config.yaml | 8 ++++---- aimm_adapters/heald_labview.py | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c3f613..c675901 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -14,16 +14,16 @@ repos: - id: debug-statements - repo: https://gitlab.com/pycqa/flake8 - rev: 4.0.1 + rev: 3.9.2 hooks: - id: flake8 - repo: https://github.com/timothycrosley/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 22.6.0 hooks: - id: black diff --git a/aimm_adapters/heald_labview.py b/aimm_adapters/heald_labview.py index 3554e5d..823963c 100644 --- a/aimm_adapters/heald_labview.py +++ b/aimm_adapters/heald_labview.py @@ -305,9 +305,11 @@ def iter_subdirectory(mapping, path, normalize=False): if normalize: norm_node = NormalizedReader(filepaths[i]) if not norm_node.is_empty(): - experiment_group[filepaths[i].stem][ - filepaths[i].name - ] = norm_node.read() + read_node = norm_node.read() + if read_node is not None: + experiment_group[filepaths[i].stem][ + filepaths[i].name + ] = read_node else: cache_key = (Path(__file__).stem, filepaths[i]) end_node = with_object_cache(cache_key, build_reader, filepaths[i]) From 9b8a3d4c6f9a13135b965b07e9526dc12765743f Mon Sep 17 00:00:00 2001 From: Juan Marulanda Date: Wed, 20 Jul 2022 17:33:52 -0400 Subject: [PATCH 2/3] Add methods to prepare local data for aimm server --- aimm_adapters/dat_reader.py | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 aimm_adapters/dat_reader.py diff --git a/aimm_adapters/dat_reader.py b/aimm_adapters/dat_reader.py new file mode 100644 index 0000000..53c46c0 --- /dev/null +++ b/aimm_adapters/dat_reader.py @@ -0,0 +1,134 @@ +from pathlib import Path + +import numpy as np +import pandas as pd + + +def readdatfiles(folderpath, dataset_name): + """ + Extract data and metadata information from a collection of dat files inside a specified folder + + + Parameters + ---------- + folderpath : string + path to folder with dat files. + dataset_name : string + name of the dataset to be used in the aimm server. + + Returns + ------- + metadata_collection : dict + Collection of metadata captured from files. Each entry is represented as {filename:dict}. + data_collection : dict + Collection of data captured from files. Each entry is represented as {filename:dataframe}. + + """ + + path = Path(folderpath) + filepaths = sorted(path.iterdir()) + + data_collection = {} + metadata_collection = {} + for filepath in filepaths: + if filepath.suffix == ".dat": + with open(filepath) as file: + lines = file.readlines() + + metadata = {} + data = [] + + metadata["dataset"] = dataset_name + metadata["fname"] = filepath.name + for line in lines: + line = line.rstrip() + if line[0] == "#": + # Metadata + meta_raw = line[2:] + if ": " in meta_raw: + # Builds metadata + meta_split = meta_raw.split(": ") + keys = meta_split[0].split(".") + if keys[0].lower() not in metadata: + metadata[keys[0].lower()] = {} + + if len(meta_split) > 1: + metadata[keys[0].lower()][keys[1].lower()] = meta_split[ + 1 + ] + else: + metadata[keys[0].lower()][keys[1].lower()] = None + + else: + # Builds column names in dataframe + meta_raw = " ".join( + meta_raw.split() + ) # Remove unwanted white spaces + columns = meta_raw.split() + + else: + # Data + sample = line.split() + sample = list(map(float, sample)) + data.append(sample) + + metadata_collection[filepath.stem] = metadata + + df = pd.DataFrame(np.array(data), columns=columns) + data_collection[filepath.stem] = df + + return data_collection, metadata_collection + + +def get_heald_data(parent_node, dataset_name, data_collection, metadata_collection): + """ + Navigates a tiled tree and searches for the nodes that meet a criteria that is compatible in the aimm server + + Parameters + ---------- + parent_node : tiled.client.node.Node + Root node of the tree. + dataset_name : str + name of the dataset that will be used in the aimm server. This will be added to the metadata of each sample + data_collection : dict + container where the data will be saved recursively. + metadata_collection : dict + container where the metadadata will be saved recursively. + + Returns + ------- + data_collection : dict + container where all the data was be saved. + metadata_collection : dict + container where all the metadata was be saved. + + """ + + from tiled.client.node import Node + + for child_node in parent_node: + if isinstance(parent_node[child_node], Node): + data_collection, metadata_collection = get_heald_data( + parent_node[child_node], + dataset_name, + data_collection, + metadata_collection, + ) + else: + if "common" in parent_node[child_node].metadata: + if ( + parent_node[child_node].metadata["common"]["element"]["symbol"] + is not None + ): + path = parent_node[child_node].path + path_name = "-".join(path) + + data_collection[path_name] = parent_node[child_node].read() + metadata_collection[path_name] = dict( + parent_node[child_node].metadata + ) + metadata_collection[path_name]["dataset"] = dataset_name + metadata_collection[path_name]["sample"] = {"name": child_node} + metadata_collection[path_name]["fname"] = path_name + + return data_collection, metadata_collection From f4327372e1b38617dff03fe77ef4ab3f66e61d27 Mon Sep 17 00:00:00 2001 From: Juan Marulanda Date: Thu, 21 Jul 2022 09:35:31 -0400 Subject: [PATCH 3/3] whitespace --- aimm_adapters/{ => scripts}/dat_reader.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename aimm_adapters/{ => scripts}/dat_reader.py (100%) diff --git a/aimm_adapters/dat_reader.py b/aimm_adapters/scripts/dat_reader.py similarity index 100% rename from aimm_adapters/dat_reader.py rename to aimm_adapters/scripts/dat_reader.py