Merge branch 'main' into treddy_derived_metrics_1

Shane Snyder · Shane Snyder · commit 6f5f716b3b72 · 2023-02-08T16:35:34.000-06:00
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -1,7 +1,7 @@
 macos_instance:
-  image: ghcr.io/cirruslabs/macos-monterey-base:latest
+  image: ghcr.io/cirruslabs/macos-ventura-base:latest
 
-task:
+macos_M1_native_apple_silicon_py310_task:
   script: |
     brew install python@3.10
     brew install python-tk@3.10
diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade pytest mypy pyflakes asv pytest-cov codecov lxml matplotlib packaging
+          python -m pip install --upgrade pytest pyflakes asv pytest-cov codecov lxml matplotlib packaging "mypy<1.0.0"
       - if: ${{matrix.platform == 'macos-latest'}}
         name: Install MacOS deps
         run: |
diff --git a/.github/workflows/runtime_ci.yml b/.github/workflows/runtime_ci.yml
@@ -26,10 +26,11 @@ jobs:
           sudo apt-get update -y
           sudo apt-get install -y hdf5-tools libhdf5-openmpi-dev openmpi-bin
           python -m pip install --upgrade pip
-          python -m pip install --upgrade pytest mpi4py
+          python -m pip install --upgrade pytest mpi4py cython numpy wheel pkgconfig "setuptools<62.0.0"
           # we need to build h5py with the system HDF5 lib backend
           export HDF5_MPI="ON"
-          CC=mpicc python -m pip install --no-binary=h5py h5py
+          # Install h5py https://github.com/h5py/h5py/issues/2222
+          CC=mpicc python -m pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation
       - name: Install darshan-runtime
         run: |
           export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/include/hdf5/openmpi/
diff --git a/ChangeLog b/ChangeLog
@@ -2,6 +2,17 @@
 Darshan Release Change Log
 --------------------------
 
+Darshan-3.4.2
+=============
+* Fixed segfault when defining scalar variables in PnetCDF module
+* Fixed memory corruption (and potential segfault) when
+  reading/writing high-dimensional PnetCDF variables using
+  vara/vars/varm interfaces
+* Fixed crashes related to using PnetCDF vard interfaces with input
+  MPI_DATATYPE_NULL datatypes
+* Fixed bug attributing all PnetCDF variable instrumentation to the
+  first variable instrumented
+
 Darshan-3.4.1
 =============
 * Added comprehensive instrumentation of PnetCDF APIs via PNETCDF_FILE and
diff --git a/configure.ac b/configure.ac
@@ -5,7 +5,8 @@ dnl Process this file with autoconf to produce a configure script.
 dnl You may need to use autoheader as well if changing any DEFINEs
 
 AC_PREREQ([2.69])
-AC_INIT([Darshan],[3.4.1],[darshan-users@lists.mcs.anl.gov],
+AC_INIT([Darshan], m4_normalize(m4_include([./darshan.version])),
+        [darshan-users@lists.mcs.anl.gov],
         [darshan],[https://www.mcs.anl.gov/research/projects/darshan])
 AC_CONFIG_HEADERS(config.h)
 AH_TOP([/*
diff --git a/darshan-runtime/configure.ac b/darshan-runtime/configure.ac
@@ -5,7 +5,8 @@ dnl Process this file with autoconf to produce a configure script.
 dnl You may need to use autoheader as well if changing any DEFINEs
 
 AC_PREREQ([2.69])
-AC_INIT([Darshan Runtime],[3.4.1],[darshan-users@lists.mcs.anl.gov],
+AC_INIT([Darshan Runtime], m4_normalize(m4_include([../darshan.version])),
+        [darshan-users@lists.mcs.anl.gov],
         [darshan-runtime],[https://www.mcs.anl.gov/research/projects/darshan])
 AC_CONFIG_HEADERS(darshan-runtime-config.h)
 AH_TOP([/*
diff --git a/darshan-test/RELEASE-CHECKLIST.txt b/darshan-test/RELEASE-CHECKLIST.txt
@@ -11,8 +11,7 @@ Notes on how to release a new version of Darshan
     - commit
  4) update the Changelog, if needed (browse git log since last release)
     - commit
- 5) update version number in configure.ac (see AC_INIT)
-    - at top-level, as well as in both darshan-util/ and darshan-runtime/
+ 5) update version number in top-level darshan.version file
     - commit
  6) follow checklist for corresponding release of PyDarshan
     - found at darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt
diff --git a/darshan-test/python_runtime_tests.py b/darshan-test/python_runtime_tests.py
@@ -1,3 +1,4 @@
+import sys
 import glob
 import subprocess
 import os
@@ -187,8 +188,9 @@ def do_forked_process_test(tmpdir, darshan_install_path):
 
     with tmpdir.as_cwd():
         cwd = os.getcwd()
+        python_exe = sys.executable
         subprocess.check_output([
-                     "python",
+                     python_exe,
                      f"{test_script_path}"],
                      env={'LD_PRELOAD': darshan_lib_path,
                           'DARSHAN_ENABLE_NONMPI': "1",
diff --git a/darshan-util/configure.ac b/darshan-util/configure.ac
@@ -5,7 +5,8 @@ dnl Process this file with autoconf to produce a configure script.
 dnl You may need to use autoheader as well if changing any DEFINEs
 
 AC_PREREQ([2.69])
-AC_INIT([Darshan Utility],[3.4.1],[darshan-users@lists.mcs.anl.gov],
+AC_INIT([Darshan Utility], m4_normalize(m4_include([../darshan.version])),
+        [darshan-users@lists.mcs.anl.gov],
         [darshan-util],[https://www.mcs.anl.gov/research/projects/darshan])
 AC_CONFIG_HEADERS(darshan-util-config.h)
 AH_TOP([/*
diff --git a/darshan-util/pydarshan/CHANGELOG.rst b/darshan-util/pydarshan/CHANGELOG.rst
@@ -1,3 +1,7 @@
+PyDarshan-3.4.2.0
+=================
+* Track Darshan 3.4.2 release, no PyDarshan changes
+
 PyDarshan-3.4.1.0
 =================
  * Fixed memory leaks in the following backend CFFI bindings
diff --git a/darshan-util/pydarshan/darshan/__init__.py b/darshan-util/pydarshan/darshan/__init__.py
@@ -3,8 +3,8 @@
 PyDarshan also provides a suite of analysis utilities.
 """
 
-__version__ = '3.4.1.0'
-__darshanutil_version__ = '3.4.1'
+__version__ = '3.4.2.0'
+__darshanutil_version__ = '3.4.2'
 
 import logging
 logger = logging.getLogger(__name__)
diff --git a/darshan-util/pydarshan/darshan/backend/cffi_backend.py b/darshan-util/pydarshan/darshan/backend/cffi_backend.py
@@ -659,6 +659,57 @@ def _log_get_heatmap_record(log):
     return rec
 
 
+def _df_to_rec(rec_dict, mod_name, rec_index_of_interest=None):
+    """
+    Pack the DataFrames-format PyDarshan data back into
+    a C buffer of records that can be consumed by darshan-util
+    C code.
+
+    Parameters
+    ----------
+    rec_dict: dict
+        Dictionary containing the counter and fcounter dataframes.
+
+    mod_name: str
+        Name of the darshan module.
+
+    rec_index_of_interest: int or None
+        If ``None``, use all records in the dataframe. Otherwise,
+        repack only the the record at the provided integer index.
+
+    Returns
+    -------
+    buf: Raw char array containing a buffer of record(s) or a single record.
+    """
+    counters_df = rec_dict["counters"]
+    fcounters_df = rec_dict["fcounters"]
+    counters_n_cols = counters_df.shape[1]
+    fcounters_n_cols = fcounters_df.shape[1]
+    if rec_index_of_interest is None:
+        num_recs = counters_df.shape[0]
+        # newer pandas versions can support ...
+        # but we use a slice for now
+        rec_index_of_interest = slice(0, counters_df.shape[0])
+    else:
+        num_recs = 1
+    # id and rank columns are duplicated
+    # in counters and fcounters
+    rec_arr = np.recarray(shape=(num_recs), dtype=[("id", "<u8", (1,)),
+                                                   ("rank", "<i8", (1,)),
+                                                   ("counters", "<i8", (counters_n_cols - 2,)),
+                                                   ("fcounters", "<f8", (fcounters_n_cols - 2,))])
+    rec_arr.fcounters = fcounters_df.iloc[rec_index_of_interest, 2:].to_numpy()
+    rec_arr.counters = counters_df.iloc[rec_index_of_interest, 2:].to_numpy()
+    if num_recs > 1:
+        rec_arr.id = counters_df.iloc[rec_index_of_interest, 0].to_numpy().reshape((num_recs, 1))
+        rec_arr.rank = counters_df.iloc[rec_index_of_interest, 1].to_numpy().reshape((num_recs, 1))
+    else:
+        rec_arr.id = counters_df.iloc[rec_index_of_interest, 0]
+        rec_arr.rank = counters_df.iloc[rec_index_of_interest, 1]
+    buf = rec_arr.tobytes()
+    return buf
+
+
 @functools.lru_cache()
 def log_get_derived_metrics(log_path: str, mod_name: str):
     """
diff --git a/darshan-util/pydarshan/darshan/cli/summary.py b/darshan-util/pydarshan/darshan/cli/summary.py
@@ -5,10 +5,7 @@
 import argparse
 import datetime
 from collections import OrderedDict
-if sys.version_info >= (3, 7):
-    import importlib.resources as importlib_resources
-else:
-    import importlib_resources
+import importlib.resources as importlib_resources
 
 from typing import Any, Union, Callable
 
@@ -415,7 +412,8 @@ def register_figures(self):
             "Average (across all ranks) amount of run time that each process "
             "spent performing I/O, broken down by access type. See the right "
             "edge bar graph on heat maps in preceding section to indicate if "
-            "I/O activity was balanced across processes."
+            "I/O activity was balanced across processes. The 'Wait' category "
+            "is only meaningful for PNETCDF asynchronous I/O operations."
         )
         io_cost_params = {
             "section_title": "Cross-Module Comparisons",
diff --git a/darshan-util/pydarshan/darshan/experimental/plots/plot_dxt_heatmap.py b/darshan-util/pydarshan/darshan/experimental/plots/plot_dxt_heatmap.py
@@ -3,6 +3,7 @@
 heatmap figure for the Darshan job summary.
 """
 
+from __future__ import annotations
 import functools
 from typing import (Any, List, Sequence, Union,
                     TYPE_CHECKING, Tuple, Optional)
@@ -12,15 +13,6 @@
 if TYPE_CHECKING:
     import numpy.typing as npt
 
-# we can't use PEP563 delayed type
-# evaluations while we have to support
-# Python 3.6 because the __future__ import is not
-# available yet;
-# TODO: delete the mocking and restore the
-# from __future__ import annotations
-from unittest.mock import MagicMock
-npt = MagicMock()
-
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
diff --git a/darshan-util/pydarshan/darshan/experimental/plots/plot_io_cost.py b/darshan-util/pydarshan/darshan/experimental/plots/plot_io_cost.py
@@ -30,7 +30,7 @@ def get_by_avg_series(df: Any, mod_key: str, nprocs: int) -> Any:
     Returns
     -------
     by_avg_series: a ``pd.Series`` containing the
-    average read, write, and meta times.
+    average read, write, meta, and wait times.
 
     """
     # filter out all except the following columns
@@ -39,11 +39,16 @@ def get_by_avg_series(df: Any, mod_key: str, nprocs: int) -> Any:
         f"{mod_key}_F_WRITE_TIME",
         f"{mod_key}_F_META_TIME",
     ]
+    if "PNETCDF_FILE" in mod_key:
+        cols.append("PNETCDF_FILE_F_WAIT_TIME")
+    else:
+        cols.append("Wait")
     by_avg_series = df.filter(cols, axis=1).sum(axis=0) / nprocs
     # reindex to ensure 3 rows are always created
     by_avg_series = by_avg_series.reindex(cols, fill_value=0.0)
     # rename the columns so the labels are automatically generated when plotting
     name_dict = {cols[0]: "Read", cols[1]: "Write", cols[2]: "Meta"}
+    name_dict[cols[3]] = "Wait"
     by_avg_series.rename(index=name_dict, inplace=True)
     return by_avg_series
 
@@ -173,6 +178,10 @@ def plot_io_cost(report: darshan.DarshanReport) -> Any:
     ax_raw.set_ylabel("Runtime (s)")
     handles, labels = ax_raw.get_legend_handles_labels()
     ax_norm.legend(handles[::-1], labels[::-1], loc="upper left", bbox_to_anchor=(1.22, 1.02))
+    # rotate the xticklabels so they don't overlap
+    for ax in [ax_raw, ax_norm]:
+        for label in ax.get_xticklabels():
+            label.set_rotation(90)
     # adjust the figure to reduce white space
     io_cost_fig.subplots_adjust(right=0.59)
     io_cost_fig.tight_layout()
diff --git a/darshan-util/pydarshan/darshan/tests/test_cffi_misc.py b/darshan-util/pydarshan/darshan/tests/test_cffi_misc.py
@@ -9,6 +9,7 @@
 from numpy.testing import assert_array_equal, assert_allclose
 import darshan
 import darshan.backend.cffi_backend as backend
+from darshan.backend.cffi_backend import ffi, libdutil, _structdefs
 from darshan.log_utils import get_log_path
 
 def test_get_lib_version():
@@ -159,3 +160,115 @@ def test_log_get_generic_record(dtype):
         # make sure the returned key/column names agree
         assert actual_counter_names == expected_counter_names
         assert actual_fcounter_names == expected_fcounter_names
+
+
+@pytest.mark.parametrize("log_name", [
+    "imbalanced-io.darshan",
+    "e3sm_io_heatmap_only.darshan",
+    ])
+@pytest.mark.parametrize("module, index", [
+    ("POSIX", 0),
+    ("POSIX", 3),
+    ("POSIX", 5),
+    ("MPI-IO", 0),
+    ("MPI-IO", 2),
+    # less records available for STDIO testing
+    # with these logs
+    ("STDIO", 0),
+    ])
+def test_df_to_rec(log_name, index, module):
+    # test for packing a dataframe into a C-style record
+    # this is perhaps nothing more than a "round-trip" test
+    log_path = get_log_path(log_name)
+    with darshan.DarshanReport(log_path, read_all=True) as report:
+        report.mod_read_all_records(module, dtype="pandas")
+        rec_dict = report.records[module][0]
+
+    # id and rank are not formally included in the reconsituted
+    # (f)counters "buffer" so truncate a bit on comparison
+    expected_fcounters = rec_dict["fcounters"].iloc[index, 2:]
+    expected_counters = rec_dict["counters"].iloc[index, 2:].astype(np.int64)
+    expected_id = rec_dict["counters"].iloc[index, 0].astype(np.uint64)
+    expected_rank = rec_dict["counters"].iloc[index, 1]
+
+    # retrive the "re-packed"/actual record data:
+    rbuf = backend._df_to_rec(rec_dict, module, index)
+    rec_buf = ffi.from_buffer(_structdefs[module].replace("**", "*"), rbuf)
+    actual_fcounters = np.frombuffer(ffi.buffer(rec_buf[0].fcounters))
+    actual_counters = np.frombuffer(ffi.buffer(rec_buf[0].counters), dtype=np.int64)
+    actual_id = rec_buf[0].base_rec.id
+    actual_rank = rec_buf[0].base_rec.rank
+
+
+    assert_allclose(actual_fcounters, expected_fcounters)
+    assert_allclose(actual_counters, expected_counters)
+    assert actual_id == expected_id
+    assert actual_rank == expected_rank
+
+
+@pytest.mark.parametrize("python_filter, expected_counts", [
+    # whether to do an initial filtering
+    # of the DataFrame in Python before
+    # packing it back into C records
+    (True, [18, 12, 2, 1]),
+    (False, [1026, 12, 2, 1]) # see gh-867
+    ])
+def test_reverse_record_array(python_filter, expected_counts):
+    # pack pandas DataFrame objects back into
+    # a contiguous buffer of several records
+    # and then use the darshan-util C lib accumulator
+    # on that record array, and compare the results
+    # with those discussed in gh-867 from Perl report
+    log_path = get_log_path("imbalanced-io.darshan")
+    with darshan.DarshanReport(log_path, read_all=True) as report:
+        nprocs = report.metadata['job']['nprocs']
+        modules = report.modules
+        report.mod_read_all_records("POSIX", dtype="pandas")
+        rec_dict = report.records["POSIX"][0]
+    counters_df = rec_dict["counters"]
+    fcounters_df = rec_dict["fcounters"]
+    if python_filter:
+        # gh-867 and the perl report filtered files that were
+        # only stat'd rather than opened, so demo the same filtering
+        # here at Python layer, then feed back to C accum stuff
+        fcounters_df = fcounters_df[counters_df["POSIX_OPENS"] > 0]
+        counters_df = counters_df[counters_df["POSIX_OPENS"] > 0]
+        rec_dict["counters"] = counters_df
+        rec_dict["fcounters"] = fcounters_df
+    num_recs = rec_dict["fcounters"].shape[0]
+    record_array = backend._df_to_rec(rec_dict, "POSIX")
+
+	# need to deal with the low-level C stuff to set up
+    # accumulator infrastructure to receive the repacked
+    # records
+    darshan_accumulator = ffi.new("darshan_accumulator *")
+    r = libdutil.darshan_accumulator_create(modules["POSIX"]['idx'],
+                                            nprocs,
+                                            darshan_accumulator)
+    assert r == 0
+    r_i = libdutil.darshan_accumulator_inject(darshan_accumulator[0], record_array, num_recs)
+    assert r_i == 0
+    derived_metrics = ffi.new("struct darshan_derived_metrics *")
+    summation_record = ffi.new(_structdefs["POSIX"].replace("**", "*"))
+    r = libdutil.darshan_accumulator_emit(darshan_accumulator[0],
+                                          derived_metrics,
+                                          summation_record)
+    assert r == 0
+    r = libdutil.darshan_accumulator_destroy(darshan_accumulator[0])
+    assert r == 0
+
+    # the indices into category_counters are pretty opaque.. we should just
+    # move everything to Python "eventually"... (also to avoid all the junk above after filtering..)
+    # 0 = total
+    # 1 = RO
+    # 2 = WO
+    # 3 = R/W
+    actual_total_files = derived_metrics.category_counters[0].count
+    actual_ro_files = derived_metrics.category_counters[1].count
+    actual_wo_files = derived_metrics.category_counters[2].count
+    actual_rw_files = derived_metrics.category_counters[3].count
+    assert_array_equal([actual_total_files,
+                        actual_ro_files,
+                        actual_wo_files,
+                        actual_rw_files],
+                        expected_counts)
diff --git a/darshan-util/pydarshan/darshan/tests/test_plot_io_cost.py b/darshan-util/pydarshan/darshan/tests/test_plot_io_cost.py
diff --git a/darshan-util/pydarshan/pyproject.toml b/darshan-util/pydarshan/pyproject.toml
diff --git a/darshan-util/pydarshan/setup.cfg b/darshan-util/pydarshan/setup.cfg
diff --git a/darshan-util/pydarshan/setup.py b/darshan-util/pydarshan/setup.py
diff --git a/darshan.version b/darshan.version
