From 526527581505e1593c9afb7ae47b8f4127e9d7d3 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:05:06 +0200 Subject: [PATCH 01/15] ruff rules --- pyproject.toml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8b0e213..f9ab3a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,3 +70,56 @@ homepage = "https://github.com/pylhc/sdds" repository = "https://github.com/pylhc/sdds" documentation = "https://pylhc.github.io/sdds/" changelog = "https://github.com/pylhc/sdds/blob/master/CHANGELOG.md" + +# ----- Dev Tools Configuration ----- # + +[tool.ruff] +exclude = [ + ".eggs", + ".git", + ".mypy_cache", + ".venv", + "_build", + "build", + "dist", +] + +# Assume Python 3.10+ +target-version = "py310" + +line-length = 100 +indent-width = 4 + +[tool.ruff.lint] +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +ignore = [ + "E501", # line too long + "FBT001", # boolean-type-hint-positional-argument + "FBT002", # boolean-default-value-positional-argument + "PT019", # pytest-fixture-param-without-value (but suggested solution fails) +] +extend-select = [ + "F", # Pyflakes rules + "W", # PyCodeStyle warnings + "E", # PyCodeStyle errors + "I", # Sort imports properly + "A", # Detect shadowed builtins + "N", # enforce naming conventions, e.g. ClassName vs function_name + "UP", # Warn if certain things can changed due to newer Python versions + "C4", # Catch incorrect use of comprehensions, dict, list, etc + "FA", # Enforce from __future__ import annotations + "FBT", # detect boolean traps + "ISC", # Good use of string concatenation + "BLE", # disallow catch-all exceptions + "ICN", # Use common import conventions + "RET", # Good return practices + "SIM", # Common simplification rules + "TID", # Some good import practices + "TC", # Enforce importing certain types in a TYPE_CHECKING block + "PTH", # Use pathlib instead of os.path + "NPY", # Some numpy-specific things +] +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] From c2b78ed6b31a627b627f7aef55c83c67a2d8afa2 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:05:44 +0200 Subject: [PATCH 02/15] formatting --- sdds/__init__.py | 1 + sdds/classes.py | 33 ++++++++++++++++++++++---- sdds/reader.py | 59 ++++++++++++++++++++++++++++++++++++---------- sdds/writer.py | 13 +++++++++- tests/test_sdds.py | 12 ++++++++-- 5 files changed, 97 insertions(+), 21 deletions(-) diff --git a/sdds/__init__.py b/sdds/__init__.py index b49a99b..c92d37a 100644 --- a/sdds/__init__.py +++ b/sdds/__init__.py @@ -1,4 +1,5 @@ """Exposes SddsFile, read_sdds and write_sdds directly in sdds namespace.""" + from sdds.classes import SddsFile from sdds.reader import read_sdds from sdds.writer import write_sdds diff --git a/sdds/classes.py b/sdds/classes.py index 6077913..652afdf 100644 --- a/sdds/classes.py +++ b/sdds/classes.py @@ -6,6 +6,7 @@ Implementation are based on documentation at: https://ops.aps.anl.gov/manuals/SDDStoolkit/SDDStoolkitsu2.html """ + import logging import warnings from dataclasses import dataclass, fields @@ -33,8 +34,24 @@ "boolean": "i1", "string": "s", } -NUMTYPES_SIZES = {"float": 4, "double": 8, "short": 2, "long": 4, "llong": 8, "char": 1, "boolean": 1} -NUMTYPES_CAST = {"float": float, "double": float, "short": int, "long": int, "llong": int, "char": str, "boolean": int} +NUMTYPES_SIZES = { + "float": 4, + "double": 8, + "short": 2, + "long": 4, + "llong": 8, + "char": 1, + "boolean": 1, +} +NUMTYPES_CAST = { + "float": float, + "double": float, + "short": int, + "long": int, + "llong": int, + "char": str, + "boolean": int, +} def get_dtype_str(type_: str, endianness: str = "big", length: Optional[int] = None): @@ -149,7 +166,9 @@ def __post_init__(self): # all is fine continue - LOGGER.debug(f"converting {field.name}: " f"{type(value).__name__} -> {hinted_type.__name__}") + LOGGER.debug( + f"converting {field.name}: {type(value).__name__} -> {hinted_type.__name__}" + ) setattr(self, field.name, hinted_type(value)) def get_key_value_string(self) -> str: @@ -158,7 +177,9 @@ def get_key_value_string(self) -> str: Hint: `ClassVars` (like ``TAG``) are ignored in `fields`. """ field_values = {field.name: getattr(self, field.name) for field in fields(self)} - return ", ".join([f"{key}={value}" for key, value in field_values.items() if value is not None]) + return ", ".join( + [f"{key}={value}" for key, value in field_values.items() if value is not None] + ) def __repr__(self): return f"" @@ -298,7 +319,9 @@ def __init__( self.description = description self.definitions = {definition.name: definition for definition in definitions_list} - self.values = {definition.name: value for definition, value in zip(definitions_list, values_list)} + self.values = { + definition.name: value for definition, value in zip(definitions_list, values_list) + } def __getitem__(self, name: str) -> Tuple[Definition, Any]: return self.definitions[name], self.values[name] diff --git a/sdds/reader.py b/sdds/reader.py index d48e496..52ff450 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -5,6 +5,7 @@ This module contains the reading functionality of ``sdds``. It provides a high-level function to read SDDS files in different formats, and a series of helpers. """ + import gzip import os import pathlib @@ -17,9 +18,19 @@ import numpy as np -from sdds.classes import (ENCODING, NUMTYPES_CAST, NUMTYPES_SIZES, Array, - Column, Data, Definition, Description, Parameter, - SddsFile, get_dtype_str) +from sdds.classes import ( + ENCODING, + NUMTYPES_CAST, + NUMTYPES_SIZES, + Array, + Column, + Data, + Definition, + Description, + Parameter, + SddsFile, + get_dtype_str, +) # ----- Providing Opener Abstractions for the Reader ----- # @@ -115,7 +126,9 @@ def _read_header( ) -> Tuple[str, List[Definition], Optional[Description], Data]: word_gen = _gen_words(inbytes) version = next(word_gen) # First token is the SDDS version - assert version == "SDDS1", "This module is compatible with SDDS v1 only... are there really other versions?" + assert version == "SDDS1", ( + "This module is compatible with SDDS v1 only... are there really other versions?" + ) definitions: List[Definition] = [] description: Optional[Description] = None data: Optional[Data] = None @@ -152,13 +165,17 @@ def _sort_definitions(orig_defs: List[Definition]) -> List[Definition]: According to the specification, parameters appear first in data pages then arrays and then columns. Inside each group they follow the order of appearance in the header. """ - definitions: List[Definition] = [definition for definition in orig_defs if isinstance(definition, Parameter)] + definitions: List[Definition] = [ + definition for definition in orig_defs if isinstance(definition, Parameter) + ] definitions.extend([definition for definition in orig_defs if isinstance(definition, Array)]) definitions.extend([definition for definition in orig_defs if isinstance(definition, Column)]) return definitions -def _read_data(data: Data, definitions: List[Definition], inbytes: IO[bytes], endianness: str) -> List[Any]: +def _read_data( + data: Data, definitions: List[Definition], inbytes: IO[bytes], endianness: str +) -> List[Any]: if data.mode == "binary": return _read_data_binary(definitions, inbytes, endianness) elif data.mode == "ascii": @@ -172,17 +189,24 @@ def _read_data(data: Data, definitions: List[Definition], inbytes: IO[bytes], en ############################################################################## -def _read_data_binary(definitions: List[Definition], inbytes: IO[bytes], endianness: str) -> List[Any]: +def _read_data_binary( + definitions: List[Definition], inbytes: IO[bytes], endianness: str +) -> List[Any]: row_count: int = _read_bin_int(inbytes, endianness) # First int in bin data functs_dict: Dict[Type[Definition], Callable] = { Parameter: _read_bin_param, Column: lambda x, y, z: _read_bin_column(x, y, z, row_count), Array: _read_bin_array, } - return [functs_dict[definition.__class__](inbytes, definition, endianness) for definition in definitions] + return [ + functs_dict[definition.__class__](inbytes, definition, endianness) + for definition in definitions + ] -def _read_bin_param(inbytes: IO[bytes], definition: Parameter, endianness: str) -> Union[int, float, str]: +def _read_bin_param( + inbytes: IO[bytes], definition: Parameter, endianness: str +) -> Union[int, float, str]: try: if definition.fixed_value is not None: if definition.type == "string": @@ -193,7 +217,9 @@ def _read_bin_param(inbytes: IO[bytes], definition: Parameter, endianness: str) if definition.type == "string": str_len: int = _read_bin_int(inbytes, endianness) return _read_string(inbytes, str_len, endianness) - return NUMTYPES_CAST[definition.type](_read_bin_numeric(inbytes, definition.type, 1, endianness)[0]) + return NUMTYPES_CAST[definition.type]( + _read_bin_numeric(inbytes, definition.type, 1, endianness)[0] + ) def _read_bin_column(inbytes: IO[bytes], definition: Column, endianness: str, row_count: int): @@ -216,7 +242,9 @@ def _read_bin_array(inbytes: IO[bytes], definition: Array, endianness: str) -> A return data.reshape(dims) -def _read_bin_array_len(inbytes: IO[bytes], num_dims: Optional[int], endianness: str) -> Tuple[List[int], int]: +def _read_bin_array_len( + inbytes: IO[bytes], num_dims: Optional[int], endianness: str +) -> Tuple[List[int], int]: if num_dims is None: num_dims = 1 @@ -272,7 +300,9 @@ def _ascii_generator(ascii_text): return data -def _read_ascii_parameter(ascii_gen: Generator[str, None, None], definition: Parameter) -> Union[str, int, float]: +def _read_ascii_parameter( + ascii_gen: Generator[str, None, None], definition: Parameter +) -> Union[str, int, float]: # Check if we got fixed values, no need to read a line if that's the case if definition.fixed_value is not None: if definition.type == "string": @@ -359,6 +389,9 @@ def _get_def_as_dict(word_gen: Generator[str, None, None]) -> Dict[str, str]: if word.strip() == "&end": recomposed: str = " ".join(raw_str) parts = [assign for assign in recomposed.split(",") if assign] - return {key.strip(): value.strip() for (key, value) in [assign.split("=") for assign in parts]} + return { + key.strip(): value.strip() + for (key, value) in [assign.split("=") for assign in parts] + } raw_str.append(word.strip()) raise ValueError("EOF found while looking for &end tag.") diff --git a/sdds/writer.py b/sdds/writer.py index 17d1f55..26b701c 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -5,13 +5,24 @@ This module contains the writing functionality of ``sdds``. It provides a high-level function to write SDDS files in different formats, and a series of helpers. """ + import pathlib import struct from typing import IO, Any, Iterable, List, Tuple, Union import numpy as np -from sdds.classes import ENCODING, Array, Column, Data, Definition, Description, Parameter, SddsFile, get_dtype_str +from sdds.classes import ( + ENCODING, + Array, + Column, + Data, + Definition, + Description, + Parameter, + SddsFile, + get_dtype_str, +) def write_sdds(sdds_file: SddsFile, output_path: Union[pathlib.Path, str]) -> None: diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 8d6fd35..9c7f887 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -20,7 +20,15 @@ SddsFile, get_dtype_str, ) -from sdds.reader import _gen_words, _get_def_as_dict, _read_data, _read_header, _sort_definitions, gzip_open, read_sdds +from sdds.reader import ( + _gen_words, + _get_def_as_dict, + _read_data, + _read_header, + _sort_definitions, + gzip_open, + read_sdds, +) from sdds.writer import _sdds_def_as_str, write_sdds CURRENT_DIR = pathlib.Path(__file__).parent @@ -187,7 +195,7 @@ def test_read_header_optionals(self): def test_def_as_dict(): - test_str = b"test1=value1, test2= value2, \n" b"test3=value3, &end" + test_str = b"test1=value1, test2= value2, \ntest3=value3, &end" word_gen = _gen_words(io.BytesIO(test_str)) def_dict = _get_def_as_dict(word_gen) assert def_dict["test1"] == "value1" From 0312ef1806242c62aa60766f14dc9402ad00709b Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:06:29 +0200 Subject: [PATCH 03/15] simple automated fixes --- doc/conf.py | 1 - sdds/classes.py | 43 +++++++++++++++++++------------------- sdds/reader.py | 52 +++++++++++++++++++++++----------------------- sdds/writer.py | 26 +++++++++++------------ tests/test_sdds.py | 3 +-- 5 files changed, 62 insertions(+), 63 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 0a4fbdb..41a30b2 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # sdds documentation build configuration file, created by # sphinx-quickstart on Tue Feb 6 12:10:18 2018. diff --git a/sdds/classes.py b/sdds/classes.py index 652afdf..7d970cd 100644 --- a/sdds/classes.py +++ b/sdds/classes.py @@ -9,8 +9,9 @@ import logging import warnings +from collections.abc import Iterator from dataclasses import dataclass, fields -from typing import Any, ClassVar, Dict, Iterator, List, Optional, Tuple +from typing import Any, ClassVar LOGGER = logging.getLogger(__name__) @@ -54,7 +55,7 @@ } -def get_dtype_str(type_: str, endianness: str = "big", length: Optional[int] = None): +def get_dtype_str(type_: str, endianness: str = "big", length: int | None = None): return f"{ENDIAN[endianness]}{length if length is not None else ''}{NUMTYPES[type_]}" @@ -79,8 +80,8 @@ class Description: contents (str): Optional. Formal specification of the type of data stored in a data set. """ - text: Optional[str] = None - contents: Optional[str] = None + text: str | None = None + contents: str | None = None TAG: ClassVar[str] = "&description" def __repr__(self): @@ -137,11 +138,11 @@ class Definition: name: str type: str - symbol: Optional[str] = None - units: Optional[str] = None - description: Optional[str] = None - format_string: Optional[str] = None - TAG: ClassVar[Optional[str]] = None + symbol: str | None = None + units: str | None = None + description: str | None = None + format_string: str | None = None + TAG: ClassVar[str | None] = None def __post_init__(self): # Fix types (probably strings from reading files) by using the type-hints @@ -217,7 +218,7 @@ class Parameter(Definition): """ TAG: ClassVar[str] = "¶meter" - fixed_value: Optional[str] = None + fixed_value: str | None = None @dataclass @@ -240,9 +241,9 @@ class Array(Definition): """ TAG: ClassVar[str] = "&array" - field_length: Optional[int] = None - group_name: Optional[str] = None - dimensions: Optional[int] = None + field_length: int | None = None + group_name: str | None = None + dimensions: int | None = None @dataclass @@ -300,16 +301,16 @@ class SddsFile: """ version: str # This should always be "SDDS1" - description: Optional[Description] - definitions: Dict[str, Definition] - values: Dict[str, Any] + description: Description | None + definitions: dict[str, Definition] + values: dict[str, Any] def __init__( self, version: str, - description: Optional[Description], - definitions_list: List[Definition], - values_list: List[Any], + description: Description | None, + definitions_list: list[Definition], + values_list: list[Any], ) -> None: self.version = version @@ -323,10 +324,10 @@ def __init__( definition.name: value for definition, value in zip(definitions_list, values_list) } - def __getitem__(self, name: str) -> Tuple[Definition, Any]: + def __getitem__(self, name: str) -> tuple[Definition, Any]: return self.definitions[name], self.values[name] - def __iter__(self) -> Iterator[Tuple[Definition, Any]]: + def __iter__(self) -> Iterator[tuple[Definition, Any]]: for def_name in self.definitions: yield self[def_name] diff --git a/sdds/reader.py b/sdds/reader.py index 52ff450..e0ea3d2 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -11,10 +11,10 @@ import pathlib import struct import sys -from collections.abc import Callable +from collections.abc import Callable, Generator from contextlib import AbstractContextManager from functools import partial -from typing import IO, Any, Dict, Generator, List, Optional, Tuple, Type, Union +from typing import IO, Any import numpy as np @@ -51,8 +51,8 @@ def read_sdds( - file_path: Union[pathlib.Path, str], - endianness: Optional[str] = None, + file_path: pathlib.Path | str, + endianness: str | None = None, opener: OpenerType = binary_open, ) -> SddsFile: """ @@ -123,17 +123,17 @@ def read_sdds( def _read_header( inbytes: IO[bytes], -) -> Tuple[str, List[Definition], Optional[Description], Data]: +) -> tuple[str, list[Definition], Description | None, Data]: word_gen = _gen_words(inbytes) version = next(word_gen) # First token is the SDDS version assert version == "SDDS1", ( "This module is compatible with SDDS v1 only... are there really other versions?" ) - definitions: List[Definition] = [] - description: Optional[Description] = None - data: Optional[Data] = None + definitions: list[Definition] = [] + description: Description | None = None + data: Data | None = None for word in word_gen: - def_dict: Dict[str, str] = _get_def_as_dict(word_gen) + def_dict: dict[str, str] = _get_def_as_dict(word_gen) if word in (Column.TAG, Parameter.TAG, Array.TAG): definitions.append( {Column.TAG: Column, Parameter.TAG: Parameter, Array.TAG: Array}[word]( @@ -159,13 +159,13 @@ def _read_header( return version, definitions, description, data -def _sort_definitions(orig_defs: List[Definition]) -> List[Definition]: +def _sort_definitions(orig_defs: list[Definition]) -> list[Definition]: """ Sorts the definitions in the parameter, array, column order. According to the specification, parameters appear first in data pages then arrays and then columns. Inside each group they follow the order of appearance in the header. """ - definitions: List[Definition] = [ + definitions: list[Definition] = [ definition for definition in orig_defs if isinstance(definition, Parameter) ] definitions.extend([definition for definition in orig_defs if isinstance(definition, Array)]) @@ -174,11 +174,11 @@ def _sort_definitions(orig_defs: List[Definition]) -> List[Definition]: def _read_data( - data: Data, definitions: List[Definition], inbytes: IO[bytes], endianness: str -) -> List[Any]: + data: Data, definitions: list[Definition], inbytes: IO[bytes], endianness: str +) -> list[Any]: if data.mode == "binary": return _read_data_binary(definitions, inbytes, endianness) - elif data.mode == "ascii": + if data.mode == "ascii": return _read_data_ascii(definitions, inbytes) raise ValueError(f"Unsupported data mode {data.mode}.") @@ -190,10 +190,10 @@ def _read_data( def _read_data_binary( - definitions: List[Definition], inbytes: IO[bytes], endianness: str -) -> List[Any]: + definitions: list[Definition], inbytes: IO[bytes], endianness: str +) -> list[Any]: row_count: int = _read_bin_int(inbytes, endianness) # First int in bin data - functs_dict: Dict[Type[Definition], Callable] = { + functs_dict: dict[type[Definition], Callable] = { Parameter: _read_bin_param, Column: lambda x, y, z: _read_bin_column(x, y, z, row_count), Array: _read_bin_array, @@ -206,7 +206,7 @@ def _read_data_binary( def _read_bin_param( inbytes: IO[bytes], definition: Parameter, endianness: str -) -> Union[int, float, str]: +) -> int | float | str: try: if definition.fixed_value is not None: if definition.type == "string": @@ -243,8 +243,8 @@ def _read_bin_array(inbytes: IO[bytes], definition: Array, endianness: str) -> A def _read_bin_array_len( - inbytes: IO[bytes], num_dims: Optional[int], endianness: str -) -> Tuple[List[int], int]: + inbytes: IO[bytes], num_dims: int | None, endianness: str +) -> tuple[list[int], int]: if num_dims is None: num_dims = 1 @@ -274,7 +274,7 @@ def _read_string(inbytes: IO[bytes], str_len: int, endianness: str) -> str: ############################################################################## -def _read_data_ascii(definitions: List[Definition], inbytes: IO[bytes]) -> List[Any]: +def _read_data_ascii(definitions: list[Definition], inbytes: IO[bytes]) -> list[Any]: def _ascii_generator(ascii_text): for line in ascii_text: yield line @@ -288,7 +288,7 @@ def _ascii_generator(ascii_text): ascii_gen = _ascii_generator(ascii_text) # Iterate through every parameters and arrays in the file - data: List[Any] = [] + data: list[Any] = [] for definition in definitions: # Call the function handling the tag we're on # Change the current line according to the tag and dimensions @@ -302,7 +302,7 @@ def _ascii_generator(ascii_text): def _read_ascii_parameter( ascii_gen: Generator[str, None, None], definition: Parameter -) -> Union[str, int, float]: +) -> str | int | float: # Check if we got fixed values, no need to read a line if that's the case if definition.fixed_value is not None: if definition.type == "string": @@ -327,7 +327,7 @@ def _read_ascii_array(ascii_gen: Generator[str, None, None], definition: Array) dimensions = np.array(next(ascii_gen).split(), dtype="int") # Get all the data given by the dimensions - data: List[str] = [] + data: list[str] = [] while len(data) != np.prod(dimensions): # The values on each line are split by a space data += next(ascii_gen).strip().split(" ") @@ -383,8 +383,8 @@ def _gen_words(inbytes: IO[bytes]) -> Generator[str, None, None]: return -def _get_def_as_dict(word_gen: Generator[str, None, None]) -> Dict[str, str]: - raw_str: List[str] = [] +def _get_def_as_dict(word_gen: Generator[str, None, None]) -> dict[str, str]: + raw_str: list[str] = [] for word in word_gen: if word.strip() == "&end": recomposed: str = " ".join(raw_str) diff --git a/sdds/writer.py b/sdds/writer.py index 26b701c..df9c349 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -8,7 +8,8 @@ import pathlib import struct -from typing import IO, Any, Iterable, List, Tuple, Union +from collections.abc import Iterable +from typing import IO, Any import numpy as np @@ -25,7 +26,7 @@ ) -def write_sdds(sdds_file: SddsFile, output_path: Union[pathlib.Path, str]) -> None: +def write_sdds(sdds_file: SddsFile, output_path: pathlib.Path | str) -> None: """ Writes SddsFile object into ``output_path``. The byteorder will be big-endian, independent of the byteorder of the current machine. @@ -41,7 +42,7 @@ def write_sdds(sdds_file: SddsFile, output_path: Union[pathlib.Path, str]) -> No _write_data(names, sdds_file, outbytes) -def _write_header(sdds_file: SddsFile, outbytes: IO[bytes]) -> List[str]: +def _write_header(sdds_file: SddsFile, outbytes: IO[bytes]) -> list[str]: outbytes.writelines(("SDDS1\n".encode(ENCODING), "!# big-endian\n".encode(ENCODING))) names = [] if sdds_file.description is not None: @@ -55,17 +56,17 @@ def _write_header(sdds_file: SddsFile, outbytes: IO[bytes]) -> List[str]: return names -def _sdds_def_as_str(definition: Union[Description, Definition, Data]) -> str: +def _sdds_def_as_str(definition: Description | Definition | Data) -> str: return f"{definition.TAG} {definition.get_key_value_string()} &end\n" -def _write_data(names: List[str], sdds_file: SddsFile, outbytes: IO[bytes]) -> None: +def _write_data(names: list[str], sdds_file: SddsFile, outbytes: IO[bytes]) -> None: # row_count: outbytes.write(np.array(0, dtype=get_dtype_str("long")).tobytes()) - parameters: List[Tuple[Parameter, Any]] = [] - arrays: List[Tuple[Array, Any]] = [] - columns: List[Tuple[Column, Any]] = [] + parameters: list[tuple[Parameter, Any]] = [] + arrays: list[tuple[Array, Any]] = [] + columns: list[tuple[Column, Any]] = [] for name in names: if isinstance(sdds_file[name][0], Parameter): parameters.append(sdds_file[name]) # type: ignore @@ -78,7 +79,7 @@ def _write_data(names: List[str], sdds_file: SddsFile, outbytes: IO[bytes]) -> N _write_columns(columns, outbytes) -def _write_parameters(param_gen: Iterable[Tuple[Parameter, Any]], outbytes: IO[bytes]): +def _write_parameters(param_gen: Iterable[tuple[Parameter, Any]], outbytes: IO[bytes]): for param_def, value in param_gen: if param_def.type == "string": _write_string(value, outbytes) @@ -86,15 +87,14 @@ def _write_parameters(param_gen: Iterable[Tuple[Parameter, Any]], outbytes: IO[b outbytes.write(np.array(value, dtype=get_dtype_str(param_def.type)).tobytes()) -def _write_arrays(array_gen: Iterable[Tuple[Array, Any]], outbytes: IO[bytes]): +def _write_arrays(array_gen: Iterable[tuple[Array, Any]], outbytes: IO[bytes]): def get_dimensions_from_array(value): # Return the number of items per dimension # For an array a[n][m], returns [n, m] if isinstance(value, np.ndarray) or isinstance(value, list): if len(value) == 0: return [0] - else: - return [len(value)] + get_dimensions_from_array(value[0]) + return [len(value)] + get_dimensions_from_array(value[0]) return [] for array_def, value in array_gen: @@ -110,7 +110,7 @@ def get_dimensions_from_array(value): outbytes.write(np.array(value, dtype=get_dtype_str(array_def.type)).tobytes()) -def _write_columns(col_gen: Iterable[Tuple[Column, Any]], outbytes: IO[bytes]): +def _write_columns(col_gen: Iterable[tuple[Column, Any]], outbytes: IO[bytes]): # TODO: Implement the columns thing. pass diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 9c7f887..5eb6f7f 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -3,7 +3,6 @@ import pathlib import struct import sys -from typing import Dict import numpy as np import pytest @@ -339,7 +338,7 @@ def _write_read_header(): assert def_dict["type"] == original.type -def _header_from_dict(d: Dict[str, Dict[str, str]]) -> str: +def _header_from_dict(d: dict[str, dict[str, str]]) -> str: """Build a quick header from given dict.""" d = {k: v.copy() for k, v in d.items()} return ( From 4c674796c923f6a89b62aad877020602b60978e8 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:07:17 +0200 Subject: [PATCH 04/15] unnecessary list comprehension --- tests/test_sdds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 5eb6f7f..26635b4 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -228,7 +228,7 @@ def template_ascii_read_write_read(self, filepath, output): if not isinstance(value, np.ndarray): values_equal = np.isclose(value, new_val, atol=0.0001) elif isinstance(value[0], np.str_): - values_equal = all([a == b for a, b in zip(value, new_val)]) + values_equal = all(a == b for a, b in zip(value, new_val)) else: values_equal = np.isclose(value, new_val, atol=0.0001).all() From 9526cae98f99cd20182389147ccaefe48839b7b2 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:07:57 +0200 Subject: [PATCH 05/15] we dropped 3.8 so this condition is useless --- sdds/reader.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdds/reader.py b/sdds/reader.py index e0ea3d2..5398687 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -34,14 +34,7 @@ # ----- Providing Opener Abstractions for the Reader ----- # -# On Python 3.8, we cannot subscript contextlib.AbstractContextManager or collections.abc.Callable, -# which became possible with PEP 585 in Python 3.9. We will check for the runtime version and simply -# not subscript if running on 3.8. The cost here is degraded typing. -# TODO: remove this conditional once Python 3.8 has reached EoL and we drop support for it -if sys.version_info < (3, 9, 0): # we're running on 3.8, which is our lowest supported - OpenerType = Callable -else: - OpenerType = Callable[[os.PathLike], AbstractContextManager[IO]] +OpenerType = Callable[[os.PathLike], AbstractContextManager[IO]] binary_open = partial(open, mode="rb") # default opening mode, simple sdds files gzip_open = partial(gzip.open, mode="rb") # for gzip-compressed sdds files From b0ab5be2cbec3b69383c1a89ea3c0e35c00f7c33 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:08:25 +0200 Subject: [PATCH 06/15] no for loop, we can just yield from --- sdds/reader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdds/reader.py b/sdds/reader.py index 5398687..415925c 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -269,8 +269,7 @@ def _read_string(inbytes: IO[bytes], str_len: int, endianness: str) -> str: def _read_data_ascii(definitions: list[Definition], inbytes: IO[bytes]) -> list[Any]: def _ascii_generator(ascii_text): - for line in ascii_text: - yield line + yield from ascii_text # Convert bytes to ASCII, separate by lines and remove comments ascii_text = [chr(r) for r in inbytes.read()] From f8f1a7e7f77f4a709ea30ccf3f573c5321dc14d6 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:09:02 +0200 Subject: [PATCH 07/15] directly return without assignment --- sdds/reader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdds/reader.py b/sdds/reader.py index 415925c..c3e5af9 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -330,9 +330,7 @@ def _read_ascii_array(ascii_gen: Generator[str, None, None], definition: Array) # Convert to np.array so that it can be reshaped to reflect the dimensions npdata = np.array(data) - npdata = npdata.reshape(dimensions) - - return npdata + return npdata.reshape(dimensions) ############################################################################## From 11cf4fcce4666e0b45dcf0e86e3e2ce670b5b0e3 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:09:30 +0200 Subject: [PATCH 08/15] no for loop, we can just yield from --- sdds/reader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdds/reader.py b/sdds/reader.py index c3e5af9..b643714 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -368,8 +368,7 @@ def _gen_real_lines(inbytes: IO[bytes]) -> Generator[str, None, None]: def _gen_words(inbytes: IO[bytes]) -> Generator[str, None, None]: for line in _gen_real_lines(inbytes): - for word in line.split(): - yield word + yield from line.split() return From 27c18581e74d0ee2ad6463a69ca9da1d8a30e6b3 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:10:34 +0200 Subject: [PATCH 09/15] merge successive isinstance calls --- sdds/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdds/writer.py b/sdds/writer.py index df9c349..40b7fd9 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -91,7 +91,7 @@ def _write_arrays(array_gen: Iterable[tuple[Array, Any]], outbytes: IO[bytes]): def get_dimensions_from_array(value): # Return the number of items per dimension # For an array a[n][m], returns [n, m] - if isinstance(value, np.ndarray) or isinstance(value, list): + if isinstance(value, np.ndarray | list): if len(value) == 0: return [0] return [len(value)] + get_dimensions_from_array(value[0]) From 43e88da10fe20c0ef9d9de85779465e34881c039 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:12:02 +0200 Subject: [PATCH 10/15] use pathlib operations where possible --- tests/test_sdds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 26635b4..17668ef 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -361,7 +361,7 @@ def _sdds_file_pathlib() -> pathlib.Path: @pytest.fixture() def _sdds_file_str() -> str: - return os.path.join(os.path.dirname(__file__), "inputs", "test_file.sdds") + return str(CURRENT_DIR / "inputs" / "test_file.sdds") @pytest.fixture() From d66da94656c9f79bf3e357e2c0a9da0fb13bd5d9 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 10 Jul 2025 16:12:29 +0200 Subject: [PATCH 11/15] use pathlib operations where possible --- tests/test_sdds.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 17668ef..71aa03a 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -1,5 +1,4 @@ import io -import os import pathlib import struct import sys @@ -371,7 +370,7 @@ def _sdds_gzipped_file_pathlib() -> pathlib.Path: @pytest.fixture() def _sdds_gzipped_file_str() -> str: - return os.path.join(os.path.dirname(__file__), "inputs", "test_file.sdds.gz") + return str(CURRENT_DIR / "inputs" / "test_file.sdds.gz") @pytest.fixture() From c0a3e876a43040061beeebd654b25e800093bffe Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 24 Jul 2025 10:19:35 +0200 Subject: [PATCH 12/15] TYPE_CHECKING blocks where relevant --- sdds/classes.py | 8 ++++++-- sdds/reader.py | 2 ++ sdds/writer.py | 8 ++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/sdds/classes.py b/sdds/classes.py index 7d970cd..cbe35fa 100644 --- a/sdds/classes.py +++ b/sdds/classes.py @@ -7,11 +7,15 @@ https://ops.aps.anl.gov/manuals/SDDStoolkit/SDDStoolkitsu2.html """ +from __future__ import annotations + import logging import warnings -from collections.abc import Iterator from dataclasses import dataclass, fields -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar + +if TYPE_CHECKING: + from collections.abc import Iterator LOGGER = logging.getLogger(__name__) diff --git a/sdds/reader.py b/sdds/reader.py index b643714..0cb031b 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -6,6 +6,8 @@ It provides a high-level function to read SDDS files in different formats, and a series of helpers. """ +from __future__ import annotations + import gzip import os import pathlib diff --git a/sdds/writer.py b/sdds/writer.py index 40b7fd9..880d2f3 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -6,10 +6,11 @@ It provides a high-level function to write SDDS files in different formats, and a series of helpers. """ +from __future__ import annotations + import pathlib import struct -from collections.abc import Iterable -from typing import IO, Any +from typing import IO, TYPE_CHECKING, Any import numpy as np @@ -25,6 +26,9 @@ get_dtype_str, ) +if TYPE_CHECKING: + from collections.abc import Iterable + def write_sdds(sdds_file: SddsFile, output_path: pathlib.Path | str) -> None: """ From c6ead9321816f453817b6247b6d1916af11b6d1d Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 24 Jul 2025 10:27:44 +0200 Subject: [PATCH 13/15] Revert "TYPE_CHECKING blocks where relevant" This reverts commit c0a3e876a43040061beeebd654b25e800093bffe. --- sdds/classes.py | 8 ++------ sdds/reader.py | 2 -- sdds/writer.py | 8 ++------ 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/sdds/classes.py b/sdds/classes.py index cbe35fa..7d970cd 100644 --- a/sdds/classes.py +++ b/sdds/classes.py @@ -7,15 +7,11 @@ https://ops.aps.anl.gov/manuals/SDDStoolkit/SDDStoolkitsu2.html """ -from __future__ import annotations - import logging import warnings +from collections.abc import Iterator from dataclasses import dataclass, fields -from typing import TYPE_CHECKING, Any, ClassVar - -if TYPE_CHECKING: - from collections.abc import Iterator +from typing import Any, ClassVar LOGGER = logging.getLogger(__name__) diff --git a/sdds/reader.py b/sdds/reader.py index 0cb031b..b643714 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -6,8 +6,6 @@ It provides a high-level function to read SDDS files in different formats, and a series of helpers. """ -from __future__ import annotations - import gzip import os import pathlib diff --git a/sdds/writer.py b/sdds/writer.py index 880d2f3..40b7fd9 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -6,11 +6,10 @@ It provides a high-level function to write SDDS files in different formats, and a series of helpers. """ -from __future__ import annotations - import pathlib import struct -from typing import IO, TYPE_CHECKING, Any +from collections.abc import Iterable +from typing import IO, Any import numpy as np @@ -26,9 +25,6 @@ get_dtype_str, ) -if TYPE_CHECKING: - from collections.abc import Iterable - def write_sdds(sdds_file: SddsFile, output_path: pathlib.Path | str) -> None: """ From 473e87bc43b7424ce900877300c42fa89231c94d Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 24 Jul 2025 10:29:05 +0200 Subject: [PATCH 14/15] we can future annotations here --- sdds/reader.py | 2 ++ sdds/writer.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sdds/reader.py b/sdds/reader.py index b643714..b61a7de 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -6,6 +6,8 @@ It provides a high-level function to read SDDS files in different formats, and a series of helpers. """ +from __future__ import annotations # For type hints in Python < 3.10 + import gzip import os import pathlib diff --git a/sdds/writer.py b/sdds/writer.py index 40b7fd9..d1039b0 100644 --- a/sdds/writer.py +++ b/sdds/writer.py @@ -6,10 +6,11 @@ It provides a high-level function to write SDDS files in different formats, and a series of helpers. """ +from __future__ import annotations # For type hints in Python < 3.10 + import pathlib import struct -from collections.abc import Iterable -from typing import IO, Any +from typing import IO, TYPE_CHECKING, Any import numpy as np @@ -25,6 +26,9 @@ get_dtype_str, ) +if TYPE_CHECKING: + from collections.abc import Iterable + def write_sdds(sdds_file: SddsFile, output_path: pathlib.Path | str) -> None: """ From 2854e568ece3475d102c2c37775b020166fd3a05 Mon Sep 17 00:00:00 2001 From: Felix Soubelet Date: Thu, 24 Jul 2025 10:30:13 +0200 Subject: [PATCH 15/15] add warning for maintainers about the future annotations here --- sdds/classes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdds/classes.py b/sdds/classes.py index 7d970cd..f790a27 100644 --- a/sdds/classes.py +++ b/sdds/classes.py @@ -7,6 +7,10 @@ https://ops.aps.anl.gov/manuals/SDDStoolkit/SDDStoolkitsu2.html """ +# Note: do not add 'from __future__ import annotations' in this file, +# as the __post_init__ method relies on the type hints determination +# at runtime and will fail if they are all made strings (in asserts). + import logging import warnings from collections.abc import Iterator