diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d91da2b90..a48a630d0 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -35,7 +35,7 @@ jobs: run: | set -xe COMMIT_MSG=$(git log --no-merges -1 --oneline) - echo "::set-output name=message::$COMMIT_MSG" + echo "message=$COMMIT_MSG" >> $GITHUB_OUTPUT echo github.ref ${{ github.ref }} build_wheels: diff --git a/ChangeLog b/ChangeLog index d117fcefe..11c4a86ac 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,101 @@ Darshan Release Change Log -------------------------- +Darshan-3.5.0 +============= + +## Highlights of some significant changes from the previous release. ++ Darshan's documentation has moved to [readthedocs.io](https://darshan.readthedocs.io/en/stable/) ++ Support large-count MPI-IO APIs introduced in MPI 4.0. ++ Change the default mode of mmap I/O data to log file to enable. ++ Fixed an issue of capturing I/O activities of NumPy programs. ++ Adds a command-line option `--version` to print Darshan version for all utility tools. ++ Bug fixes to handling of DAOS metrics in PyDarshan and the job summary command line tool. ++ Enhancements to the job_stats and file_stats command line tools: + * Adds new performance metrics of I/O bandwidth by slowest, I/O time by slowest, and total bytes. + * Adds job and file levels of statistics for DAOS and DFS modules. + +## What's Changed +* Github action: remove setuptools version constrain by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1041 +* TYP: Disable NumPy's deprecated mypy plugin by @tylerjereddy in + https://github.com/darshan-hpc/darshan/pull/1038 +* note how to preview documentation updates by @carns in + https://github.com/darshan-hpc/darshan/pull/1042 +* clang20 cleanups by @roblatham00 in + https://github.com/darshan-hpc/darshan/pull/1035 +* revise landing page by @carns in + https://github.com/darshan-hpc/darshan/pull/1043 +* Readthedocs: note on difference between latest and stable versions by + @wkliao in https://github.com/darshan-hpc/darshan/pull/1045 +* experimenting with modifying css for table by @carns in + https://github.com/darshan-hpc/darshan/pull/1046 +* BUG: correct DAOS and DFS layout in dataframe repacking that is consumed by + the logutils accumulator API by @orcunyildiz in + https://github.com/darshan-hpc/darshan/pull/1048 +* Added I/O performance metrics to file_stats by @safaad in + https://github.com/darshan-hpc/darshan/pull/1036 +* Fix doc for Spack installation by @blastmaster in + https://github.com/darshan-hpc/darshan/pull/1050 +* add command-line option --version to print Darshan version by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1051 +* Change mmap I/O default mode to enable by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1055 +* remove duplicated entries appeared in darshan-mpiio-ld-opts by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1058 +* darshan-config: add a complete list of options configured by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1054 +* Add 4 missing MPI-IO APIs by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1059 +* release checklist: add versioning guide line by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1061 +* avoid div by zero in cli tools when there is no I/O by @carns in + https://github.com/darshan-hpc/darshan/pull/1069 +* ENH: add DAOS and DFS modules to job_stats and file_stats cli tools by + @orcunyildiz in https://github.com/darshan-hpc/darshan/pull/1065 +* support MPI-IO large count APIs by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1060 +* remove deprecated asciidoc documentation by @carns in + https://github.com/darshan-hpc/darshan/pull/1073 +* Set default MPI-IO hint cb_nodes to 1 when using OpenMPI 5.0.5 and priors by + @wkliao in https://github.com/darshan-hpc/darshan/pull/1071 +* deprecate wrapper generators by @carns in + https://github.com/darshan-hpc/darshan/pull/1039 +* Wrap fcntl() calls by @roblatham00 in + https://github.com/darshan-hpc/darshan/pull/1075 +* Strictness fix for new compilers by @bnicolae in + https://github.com/darshan-hpc/darshan/pull/1076 +* Clean up some compiler warnings by @roblatham00 in + https://github.com/darshan-hpc/darshan/pull/1078 +* Minor fixes by @wkliao in https://github.com/darshan-hpc/darshan/pull/1080 +* Setting --with-pnetcdf automatically enables enable_pnetcdf_mod by @wkliao + in https://github.com/darshan-hpc/darshan/pull/1079 +* DOC: add missing configure options to user guides by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1084 +* Add pnetcdf lib to --dyn-ld-flags by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1085 +* Patch darshan-apmpi.c and add Github CI to test autoperf by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1082 +* fix compile warning of shadowing a previous local [-Wshadow] by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1083 +* fix papi library path lookup by @wkliao in + https://github.com/darshan-hpc/darshan/pull/1086 + +## New Contributors +* @roblatham00 made their first contribution in +https://github.com/darshan-hpc/darshan/pull/1035 +* @orcunyildiz made their first contribution in +https://github.com/darshan-hpc/darshan/pull/1048 +* @safaad made their first contribution in +https://github.com/darshan-hpc/darshan/pull/1036 +* @blastmaster made their first contribution in +https://github.com/darshan-hpc/darshan/pull/1050 +* @bnicolae made their first contribution in +https://github.com/darshan-hpc/darshan/pull/1076 + +**Full Changelog**: +https://github.com/darshan-hpc/darshan/compare/3.4.7.1...3.5.0 + Darshan-3.4.7 ============= * added extensive instrumentation of DAOS's file system (DFS) and native object diff --git a/darshan-runtime/lib/Makefile.am b/darshan-runtime/lib/Makefile.am index 103a3535e..682803ece 100644 --- a/darshan-runtime/lib/Makefile.am +++ b/darshan-runtime/lib/Makefile.am @@ -105,7 +105,7 @@ if BUILD_APMPI_MODULE -I$(apmpi_root) -I$(apmpi_root)/lib endif darshan-apmpi.c: - $(LN_S) $(apmpi_root)/lib/darshan-apmpi.c . + $(LN_S) $(top_srcdir)/../modules/patched_darshan-apmpi.c $@ libdarshan_la_SOURCES = $(C_SRCS) libdarshan_la_LIBADD = -lpthread -lrt -lz -ldl $(DARSHAN_LUSTRE_LD_FLAGS) diff --git a/darshan-runtime/test/tst_runs.sh b/darshan-runtime/test/tst_runs.sh index abc62f7ab..2333f69e5 100755 --- a/darshan-runtime/test/tst_runs.sh +++ b/darshan-runtime/test/tst_runs.sh @@ -66,6 +66,12 @@ if test "x$NP" = x ; then NP=2 fi +# When TESTMPIRUN is not set, it is most likely built in the cross-compile +# environment. In this case, skip the test. +if test "x$TESTMPIRUN" = x ; then + exit 0 +fi + if test "x$HAVE_OPEN_MPI" = x1 ; then TESTMPIRUN="$TESTMPIRUN --oversubscribe" fi diff --git a/darshan-test/RELEASE-CHECKLIST.txt b/darshan-test/RELEASE-CHECKLIST.txt index 38584f60b..192959b97 100644 --- a/darshan-test/RELEASE-CHECKLIST.txt +++ b/darshan-test/RELEASE-CHECKLIST.txt @@ -10,7 +10,10 @@ Notes on how to release a new version of Darshan on the ALCF GitLab CI instance 3) make sure that documentation on Read the Docs is up to date. See instructions in readthedocs/DEVELOPER_NOTES.md - 4) update the Changelog, if needed (browse git log since last release) + 4) update file "ChangeLog", if needed (browse git log since last release) + - When using Github release mechanism, it has a nice feature to generated a + list of merged PRs, which can be copied to file ChangeLog. See 12) below. + You may want to first create a dummy release to see this note first. - commit 5) update version number in top-level darshan.version file - See below for the guideline of setting the version number. @@ -26,30 +29,50 @@ Notes on how to release a new version of Darshan ./run-all.sh 9) copy example output file (from the above testing) and commit in the darshan-logs repo in the darshan_logs/release_logs/ directory + a. log into polaris.alcf.anl.gov + b. clone darshan repo, say into $HOME/Darshan/Github/darshan + c. install Darshan, say into folder $HOME/Darshan/Github/DEV + d. cd $HOME/Darshan/Github/darshan/darshan-test/regression/ + e. ./run-all.sh $HOME/Darshan/Github/DEV $HOME/Darshan/TEST alcf-polaris-cray-module + f. clone darshan-logs repo, say into $HOME/Darshan/Github/darshan-logs + g. cd $HOME/Darshan/Github/darshan-logs/darshan_logs/release_logs + h. cp $HOME/Darshan/TEST/mpi-io-test.darshan mpi-io-test-x86_64-3.5.0.darshan + i. Create a new PR 10) make a tag for the release according to instructions at http://git-scm.com/book/en/Git-Basics-Tagging - example (annotated tag, pushed to repo): - git tag -a darshan-2.3.1 -m 'Darshan 2.3.1' - git push origin darshan-2.3.1 + git tag -a 3.5.0 -m 'Darshan 3.5.0' + git push origin 3.5.0 - NOTE: update the above to reflect the tag format needed by Read the Docs 11) manually create the release tarball - - do a fresh checkout of the Darshan release tag - git clone --branch darshan-2.3.1 \ - https://github.com/darshan-hpc/darshan.git darshan-2.3.1 - - make sure to pull in AutoPerf submodule + - Make sure to update the AutoPerf submodule git submodule update --init - - remove all .git files/directories in the top-level of the repo - - create release tarball - tar -czvf darshan-2.3.1.tar.gz /path/to/darshan-2.3.1 - - upload tarball to CELS server - scp darshan-2.3.1.tar.gz user@homes.cels.anl.gov:/nfs/pub_html/gce/projects/darshan/releases/ -12) update download page on darshan web site: www.mcs.anl.gov/darshan/download -13) update darshan-runtime and darshan-util Spack packages with new release + - Run "configure" to generate necessary autotools files + - Run "make dist", which generates a tar ball, named darshan-3.5.0.tar.gz +12) create a new release on github.com + - On Darshan repo home, click "Create a new release" on the right. + - Select Tag to be "3.5.0" + - Add release title to be "3.5.0". This must be in the form of + MAJOR.MINOR.PATCH + - Under Release notes, select Previous tag, say "3.4.7.1" + - Click "Generate release notes", which will populate a list of PRs merged + since previous tag. + - Edit the populated release note to remove or add contents if necessary. + - Click "Attach binaries" and add the release tar ball created in Step 11 + above. + - Click "Save draft" to edit later. (Drafts will not be seen by the public.) + In case a draft needs updates, use git commands below. + + `git tag -f 3.5.0 HEAD` to force an update to an existing tag. + + `git push -f origin 3.5.0` to force pushed to remote repo at github.com + + Then visit Darshan repo at github.com and edit the draft. + - When ready for the release, click "Publish release" to make a release. +13) update download page on darshan web site: www.mcs.anl.gov/darshan/download +14) update darshan-runtime and darshan-util Spack packages with new release - consider whether new variants should be exposed based on new features enabled by the release - commit, submit PR to upstream Spack repository -14) announce on web page (new post) and mailing list +15) announce on web page (new post) and mailing list ---- ## Setting the release version number diff --git a/darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt b/darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt index 4e50eb57c..9059fc435 100644 --- a/darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt +++ b/darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt @@ -7,19 +7,28 @@ Notes on how to release a new version of PyDarshan (- source venv/bin/activate) (- pip install .[dev,test] # deps for packaging, testing, and docs generation) - - Update CHANGELOG.rst - - commit + - About updating CHANGELOG.rst + - Since version 3.5.0, pydarshan released as a part of Darshan package, the + change history should go to `ChangeLog` at root folder. - Update version numbers in: darshan/__init__.py - - determine whether new job summary tool features warrant generating a new summary report to include in docs - (stored @ /nfs/pub_html/gce/projects/darshan/docs/example_report.html) + - About the pydarshan documents + - Since version 3.5.0, all Darshan documents have been moved to + readthedocs.io and are updated every time a new commit is pushed to the + main branch. - Submit to PyPI using cibuildwheel/twine: - pipx run build --sdist (in the pydarshan directory, creates a source distribution) - download wheel artifacts from GitHub, copy to 'dist/' directory created above - (click on Actions in GitHub, find most recent "Build Wheels" workflow run with artifacts) - (download artifact zip (called 'pydarshan-wheels-all'), extract wheels to 'dist/' directory) + + Click on Actions in GitHub. On the left panel, click "Build Wheels". + + Click on "Run workflow" to generate the artifacts. This will take about an hour. + + Click on the latest workflow run named "Build Wheels" + + At the bottom section, click the download icon to the right of + 'pydarshan-wheels-all' to download the artifact zip file, named + 'pydarshan-wheels-all.zip'. + + Copy the zip file to to 'dist/' directory and run command 'unzip' to + extract the wheel files. - pipx run twine upload dist/* (requires PyPI credentials for darshan project) (use '--repository testpypi' to test wheels using https://test.pypi.org/project/darshan/ before uploading to PyPI) diff --git a/darshan-util/pydarshan/darshan/__init__.py b/darshan-util/pydarshan/darshan/__init__.py index faad2a353..66832899b 100644 --- a/darshan-util/pydarshan/darshan/__init__.py +++ b/darshan-util/pydarshan/darshan/__init__.py @@ -3,8 +3,8 @@ PyDarshan also provides a suite of analysis utilities. """ -__version__ = '3.4.7.0' -__darshanutil_version__ = '3.4.7' +__version__ = '3.5.0' +__darshanutil_version__ = '3.5.0' import logging logger = logging.getLogger(__name__) diff --git a/darshan.version b/darshan.version index 2aa513199..1545d9665 100644 --- a/darshan.version +++ b/darshan.version @@ -1 +1 @@ -3.4.7 +3.5.0 diff --git a/modules/Makefile.am b/modules/Makefile.am index fbe8a4de9..1514e6848 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -3,23 +3,7 @@ # # @configure_input@ -apmpi_c_src = $(srcdir)/autoperf/apmpi/lib/darshan-apmpi.c -apmpi_c_patched = $(srcdir)/patched_darshan-apmpi.c -apmpi_c_saved = $(srcdir)/saved_darshan-apmpi.c -apmpi_c_orig = $(srcdir)/orig_darshan-apmpi.c - -# Rule to apply the patch -$(apmpi_c_saved): $(apmpi_c_src) $(apmpi_c_patched) - cp -f $(apmpi_c_src) $(apmpi_c_saved) - cp -f $(apmpi_c_patched) $(apmpi_c_src) - touch $(apmpi_c_saved) - -BUILT_SOURCES = $(apmpi_c_saved) - -EXTRA_DIST = patched_darshan-apmpi.c orig_darshan-apmpi.c - -CLEANFILES = $(apmpi_c_saved) - -clean-local: - cp -f -p $(apmpi_c_orig) $(apmpi_c_src) +EXTRA_DIST = patched_darshan-apmpi.c \ + orig_darshan-apmpi.c \ + autoperf