Prepare release of 3.5.0

.github/workflows/build_wheels.yml

@@ -35,7 +35,7 @@ jobs:
         run: |
           set -xe
           COMMIT_MSG=$(git log --no-merges -1 --oneline)
-          echo "::set-output name=message::$COMMIT_MSG"
+          echo "message=$COMMIT_MSG" >> $GITHUB_OUTPUT
           echo github.ref ${{ github.ref }}
 
   build_wheels:

ChangeLog

@@ -2,6 +2,101 @@
 Darshan Release Change Log
 --------------------------
 
+Darshan-3.5.0
+=============
+
+## Highlights of some significant changes from the previous release.
++ Darshan's documentation has moved to [readthedocs.io](https://darshan.readthedocs.io/en/stable/)
++ Support large-count MPI-IO APIs introduced in MPI 4.0.
++ Change the default mode of mmap I/O data to log file to enable.
++ Fixed an issue of capturing I/O activities of NumPy programs.
++ Adds a command-line option `--version` to print Darshan version for all utility tools.
++ Bug fixes to handling of DAOS metrics in PyDarshan and the job summary command line tool.
++ Enhancements to the job_stats and file_stats command line tools:
+  * Adds new performance metrics of I/O bandwidth by slowest, I/O time by slowest, and total bytes.
+  * Adds job and file levels of statistics for DAOS and DFS modules.
+
+## What's Changed
+* Github action: remove setuptools version constrain by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1041
+* TYP: Disable NumPy's deprecated mypy plugin by @tylerjereddy in
+  https://github.com/darshan-hpc/darshan/pull/1038
+* note how to preview documentation updates by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1042
+* clang20 cleanups by @roblatham00 in
+  https://github.com/darshan-hpc/darshan/pull/1035
+* revise landing page by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1043
+* Readthedocs: note on difference between latest and stable versions by
+  @wkliao in https://github.com/darshan-hpc/darshan/pull/1045
+* experimenting with modifying css for table by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1046
+* BUG: correct DAOS and DFS layout in dataframe repacking that is consumed by
+  the logutils accumulator API by @orcunyildiz in
+  https://github.com/darshan-hpc/darshan/pull/1048
+* Added I/O performance metrics to file_stats by @safaad in
+  https://github.com/darshan-hpc/darshan/pull/1036
+* Fix doc for Spack installation by @blastmaster in
+  https://github.com/darshan-hpc/darshan/pull/1050
+* add command-line option --version to print Darshan version by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1051
+* Change mmap I/O default mode to enable by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1055
+* remove duplicated entries appeared in darshan-mpiio-ld-opts by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1058
+* darshan-config: add a complete list of options configured by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1054
+* Add 4 missing MPI-IO APIs by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1059
+* release checklist: add versioning guide line by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1061
+* avoid div by zero in cli tools when there is no I/O by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1069
+* ENH: add DAOS and DFS modules to job_stats and file_stats cli tools by
+  @orcunyildiz in https://github.com/darshan-hpc/darshan/pull/1065
+* support MPI-IO large count APIs by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1060
+* remove deprecated asciidoc documentation by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1073
+* Set default MPI-IO hint cb_nodes to 1 when using OpenMPI 5.0.5 and priors by
+  @wkliao in https://github.com/darshan-hpc/darshan/pull/1071
+* deprecate wrapper generators by @carns in
+  https://github.com/darshan-hpc/darshan/pull/1039
+* Wrap fcntl() calls by @roblatham00 in
+  https://github.com/darshan-hpc/darshan/pull/1075
+* Strictness fix for new compilers by @bnicolae in
+  https://github.com/darshan-hpc/darshan/pull/1076
+* Clean up some compiler warnings by @roblatham00 in
+  https://github.com/darshan-hpc/darshan/pull/1078
+* Minor fixes by @wkliao in https://github.com/darshan-hpc/darshan/pull/1080
+* Setting --with-pnetcdf automatically enables enable_pnetcdf_mod by @wkliao
+  in https://github.com/darshan-hpc/darshan/pull/1079
+* DOC: add missing configure options to user guides by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1084
+* Add pnetcdf lib to --dyn-ld-flags by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1085
+* Patch darshan-apmpi.c and add Github CI to test autoperf by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1082
+* fix compile warning of shadowing a previous local [-Wshadow] by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1083
+* fix papi library path lookup by @wkliao in
+  https://github.com/darshan-hpc/darshan/pull/1086
+
+## New Contributors
+* @roblatham00 made their first contribution in
+https://github.com/darshan-hpc/darshan/pull/1035
+* @orcunyildiz made their first contribution in
+https://github.com/darshan-hpc/darshan/pull/1048
+* @safaad made their first contribution in
+https://github.com/darshan-hpc/darshan/pull/1036
+* @blastmaster made their first contribution in
+https://github.com/darshan-hpc/darshan/pull/1050
+* @bnicolae made their first contribution in
+https://github.com/darshan-hpc/darshan/pull/1076
+
+**Full Changelog**:
+https://github.com/darshan-hpc/darshan/compare/3.4.7.1...3.5.0
+
 Darshan-3.4.7
 =============
 * added extensive instrumentation of DAOS's file system (DFS) and native object

darshan-runtime/lib/Makefile.am

@@ -105,7 +105,7 @@ if BUILD_APMPI_MODULE
                   -I$(apmpi_root) -I$(apmpi_root)/lib
 endif
 darshan-apmpi.c:
-	$(LN_S) $(apmpi_root)/lib/darshan-apmpi.c .
+	$(LN_S) $(top_srcdir)/../modules/patched_darshan-apmpi.c $@
 
 libdarshan_la_SOURCES  = $(C_SRCS)
 libdarshan_la_LIBADD   = -lpthread -lrt -lz -ldl $(DARSHAN_LUSTRE_LD_FLAGS)

darshan-runtime/test/tst_runs.sh

@@ -66,6 +66,12 @@ if test "x$NP" = x ; then
    NP=2
 fi
 
+# When TESTMPIRUN is not set, it is most likely built in the cross-compile
+# environment. In this case, skip the test.
+if test "x$TESTMPIRUN" = x ; then
+   exit 0
+fi
+
 if test "x$HAVE_OPEN_MPI" = x1 ; then
    TESTMPIRUN="$TESTMPIRUN --oversubscribe"
 fi

darshan-test/RELEASE-CHECKLIST.txt

@@ -10,7 +10,10 @@ Notes on how to release a new version of Darshan
     on the ALCF GitLab CI instance
  3) make sure that documentation on Read the Docs is up to date.  See
     instructions in readthedocs/DEVELOPER_NOTES.md
- 4) update the Changelog, if needed (browse git log since last release)
+ 4) update file "ChangeLog", if needed (browse git log since last release)
+    - When using Github release mechanism, it has a nice feature to generated a
+      list of merged PRs, which can be copied to file ChangeLog. See 12) below.
+      You may want to first create a dummy release to see this note first.
     - commit
  5) update version number in top-level darshan.version file
     - See below for the guideline of setting the version number.
@@ -26,30 +29,50 @@ Notes on how to release a new version of Darshan
       ./run-all.sh <darshan_install_prefix> <scratch_dir> <system_env_profile>
  9) copy example output file (from the above testing) and commit in
     the darshan-logs repo in the darshan_logs/release_logs/ directory
+    a. log into polaris.alcf.anl.gov
+    b. clone darshan repo, say into $HOME/Darshan/Github/darshan
+    c. install Darshan, say into folder $HOME/Darshan/Github/DEV
+    d. cd $HOME/Darshan/Github/darshan/darshan-test/regression/
+    e. ./run-all.sh $HOME/Darshan/Github/DEV $HOME/Darshan/TEST alcf-polaris-cray-module
+    f. clone darshan-logs repo, say into $HOME/Darshan/Github/darshan-logs
+    g. cd $HOME/Darshan/Github/darshan-logs/darshan_logs/release_logs
+    h. cp $HOME/Darshan/TEST/mpi-io-test.darshan mpi-io-test-x86_64-3.5.0.darshan
+    i. Create a new PR
 10) make a tag for the release according to instructions at
     http://git-scm.com/book/en/Git-Basics-Tagging
     - example (annotated tag, pushed to repo):
-      git tag -a darshan-2.3.1 -m 'Darshan 2.3.1'
-      git push origin darshan-2.3.1
+      git tag -a 3.5.0 -m 'Darshan 3.5.0'
+      git push origin 3.5.0
     - NOTE: update the above to reflect the tag format needed by Read the
       Docs
 11) manually create the release tarball
-    - do a fresh checkout of the Darshan release tag
-      git clone --branch darshan-2.3.1 \
-      https://github.com/darshan-hpc/darshan.git darshan-2.3.1
-    - make sure to pull in AutoPerf submodule
+    - Make sure to update the AutoPerf submodule
       git submodule update --init
-    - remove all .git files/directories in the top-level of the repo
-    - create release tarball
-      tar -czvf darshan-2.3.1.tar.gz /path/to/darshan-2.3.1
-    - upload tarball to CELS server
-      scp darshan-2.3.1.tar.gz [email protected]:/nfs/pub_html/gce/projects/darshan/releases/
-12) update download page on darshan web site: www.mcs.anl.gov/darshan/download
-13) update darshan-runtime and darshan-util Spack packages with new release
+    - Run "configure" to generate necessary autotools files
+    - Run "make dist", which generates a tar ball, named darshan-3.5.0.tar.gz
+12) create a new release on github.com
+    - On Darshan repo home, click "Create a new release" on the right.
+    - Select Tag to be "3.5.0"
+    - Add release title to be "3.5.0". This must be in the form of
+      MAJOR.MINOR.PATCH
+    - Under Release notes, select Previous tag, say "3.4.7.1"
+    - Click "Generate release notes", which will populate a list of PRs merged
+      since previous tag.
+    - Edit the populated release note to remove or add contents if necessary.
+    - Click "Attach binaries" and add the release tar ball created in Step 11
+      above.
+    - Click "Save draft" to edit later. (Drafts will not be seen by the public.)
+      In case a draft needs updates, use git commands below.
+      + `git tag -f 3.5.0 HEAD` to force an update to an existing tag.
+      + `git push -f origin 3.5.0` to force pushed to remote repo at github.com
+      + Then visit Darshan repo at github.com and edit the draft.
+    - When ready for the release, click "Publish release" to make a release.
+13) update download page on darshan web site: www.mcs.anl.gov/darshan/download
+14) update darshan-runtime and darshan-util Spack packages with new release
     - consider whether new variants should be exposed based on new features
       enabled by the release
     - commit, submit PR to upstream Spack repository
-14) announce on web page (new post) and mailing list
+15) announce on web page (new post) and mailing list
 
 ----
 ## Setting the release version number

darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt

@@ -7,19 +7,28 @@ Notes on how to release a new version of PyDarshan
     (- source venv/bin/activate)
     (- pip install .[dev,test]    # deps for packaging, testing, and docs generation)
 
- - Update CHANGELOG.rst
-    - commit
+ - About updating CHANGELOG.rst
+    - Since version 3.5.0, pydarshan released as a part of Darshan package, the
+      change history should go to `ChangeLog` at root folder.
  - Update version numbers in:
     darshan/__init__.py
- - determine whether new job summary tool features warrant generating a new summary report to include in docs
-     (stored @ /nfs/pub_html/gce/projects/darshan/docs/example_report.html)
+ - About the pydarshan documents
+    - Since version 3.5.0, all Darshan documents have been moved to
+      readthedocs.io and are updated every time a new commit is pushed to the
+      main branch.
 
  - Submit to PyPI using cibuildwheel/twine:
     - pipx run build --sdist
         (in the pydarshan directory, creates a source distribution)
     - download wheel artifacts from GitHub, copy to 'dist/' directory created above
-        (click on Actions in GitHub, find most recent "Build Wheels" workflow run with artifacts)
-        (download artifact zip (called 'pydarshan-wheels-all'), extract wheels to 'dist/' directory)
+        + Click on Actions in GitHub. On the left panel, click "Build Wheels".
+        + Click on "Run workflow" to generate the artifacts. This will take about an hour.
+        + Click on the latest workflow run named "Build Wheels"
+        + At the bottom section, click the download icon to the right of
+          'pydarshan-wheels-all' to download the artifact zip file, named
+          'pydarshan-wheels-all.zip'.
+        + Copy the zip file to to 'dist/' directory and run command 'unzip' to
+          extract the wheel files.
     - pipx run twine upload dist/*
         (requires PyPI credentials for darshan project)
         (use '--repository testpypi' to test wheels using https://test.pypi.org/project/darshan/ before uploading to PyPI)

darshan-util/pydarshan/darshan/__init__.py

@@ -3,8 +3,8 @@
 PyDarshan also provides a suite of analysis utilities.
 """
 
-__version__ = '3.4.7.0'
-__darshanutil_version__ = '3.4.7'
+__version__ = '3.5.0'
+__darshanutil_version__ = '3.5.0'
 
 import logging
 logger = logging.getLogger(__name__)

darshan.version

@@ -1 +1 @@
-3.4.7
+3.5.0

modules/Makefile.am

@@ -3,23 +3,7 @@
 #
 # @configure_input@
 
-apmpi_c_src = $(srcdir)/autoperf/apmpi/lib/darshan-apmpi.c
-apmpi_c_patched = $(srcdir)/patched_darshan-apmpi.c
-apmpi_c_saved = $(srcdir)/saved_darshan-apmpi.c
-apmpi_c_orig = $(srcdir)/orig_darshan-apmpi.c
-
-# Rule to apply the patch
-$(apmpi_c_saved): $(apmpi_c_src) $(apmpi_c_patched)
-	cp -f $(apmpi_c_src) $(apmpi_c_saved)
-	cp -f $(apmpi_c_patched) $(apmpi_c_src)
-	touch $(apmpi_c_saved)
-
-BUILT_SOURCES = $(apmpi_c_saved)
-
-EXTRA_DIST = patched_darshan-apmpi.c orig_darshan-apmpi.c
-
-CLEANFILES = $(apmpi_c_saved)
-
-clean-local:
-	cp -f -p $(apmpi_c_orig) $(apmpi_c_src)
+EXTRA_DIST = patched_darshan-apmpi.c \
+             orig_darshan-apmpi.c \
+             autoperf