Update the HDF5 and NetCDF building blocks (#503)

- Bump the default HDF5 version to 1.14.5
- Add required libxml2 dependency to NetCDF
- Workaround issue when building NetCDF Fortran with the NVIDIA HPC SDK

Author: samcmill

docs/building_blocks.md

@@ -1412,7 +1412,7 @@ non-default compilers or other toolchain options are needed.  The
 default is empty.
 
 - __version__: The version of HDF5 source to download.  This value is
-ignored if `directory` is set.  The default value is `1.12.0`.
+ignored if `directory` is set.  The default value is `1.14.5`.
 
 - __with_PACKAGE[=ARG]__: Flags to control optional packages when
 configuring.  For instance, `with_foo=True` maps to `--with-foo`
@@ -2988,10 +2988,10 @@ directory. The default value is False.
 
 - __ospackages__: List of OS packages to install prior to configuring
 and building.  For Ubuntu, the default values are
-`ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`,
-`wget`, and `zlib1g-dev`.  For RHEL-based Linux distributions the
-default values are `ca-certificates`, `file`, `libcurl-devel`
-`m4`, `make`, `wget`, and `zlib-devel`.
+`ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`,
+`make`, `wget`, and `zlib1g-dev`.  For RHEL-based Linux distributions the
+default values are `ca-certificates`, `file`, `libcurl-devel`,
+`libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`.
 
 - __prefix__: The top level install location.  The default location is
 `/usr/local/netcdf`.

hpccm/building_blocks/hdf5.py

@@ -21,6 +21,8 @@
 from __future__ import unicode_literals
 from __future__ import print_function
 
+from packaging.version import Version
+
 import posixpath
 import re
 from copy import copy as _copy
@@ -92,7 +94,7 @@ class hdf5(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
     default is empty.
 
     version: The version of HDF5 source to download.  This value is
-    ignored if `directory` is set.  The default value is `1.12.0`.
+    ignored if `directory` is set.  The default value is `1.14.5`.
 
     with_PACKAGE[=ARG]: Flags to control optional packages when
     configuring.  For instance, `with_foo=True` maps to `--with-foo`
@@ -132,7 +134,7 @@ def __init__(self, **kwargs):
 
         super(hdf5, self).__init__(**kwargs)
 
-        self.__baseurl = kwargs.pop('baseurl', 'https://support.hdfgroup.org/ftp/HDF5/releases')
+        self.__baseurl = kwargs.pop('baseurl', False)
         self.__check = kwargs.pop('check', False)
         self.__configure_opts = kwargs.pop('configure_opts',
                                            ['--enable-cxx',
@@ -143,7 +145,14 @@ def __init__(self, **kwargs):
         # without impacting the original
         self.__toolchain = _copy(kwargs.pop('toolchain', toolchain()))
         self.__runtime_ospackages = [] # Filled in by __distro()
-        self.__version = kwargs.pop('version', '1.12.0')
+        self.__version = kwargs.pop('version', '1.14.5')
+
+        if not self.__baseurl:
+            # Download path changed with version 1.14
+            if Version(self.__version) >= Version('1.14'):
+                self.__baseurl = 'https://support.hdfgroup.org/releases/hdf5'
+            else:
+                self.__baseurl = 'https://support.hdfgroup.org/ftp/HDF5/releases'
 
         # Set the Linux distribution specific parameters
         self.__distro()
@@ -214,11 +223,19 @@ def __download(self):
         # path and the tarball contains MAJOR.MINOR.REVISION, so pull
         # apart the full version to get the MAJOR and MINOR components.
         match = re.match(r'(?P<major>\d+)\.(?P<minor>\d+)', self.__version)
-        major_minor = '{0}.{1}'.format(match.groupdict()['major'],
-                                       match.groupdict()['minor'])
-        tarball = 'hdf5-{}.tar.bz2'.format(self.__version)
-        self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format(
-            self.__baseurl, major_minor, self.__version, tarball)
+        if Version(self.__version) >= Version('1.14'):
+            major_minor = 'v{0}_{1}'.format(match.groupdict()['major'],
+                                            match.groupdict()['minor'])
+            tarball = 'hdf5-{}.tar.gz'.format(self.__version)
+            self.__url = '{0}/{1}/v{2}/downloads/{3}'.format(
+                self.__baseurl, major_minor, self.__version.replace('.', '_'),
+                tarball)
+        else:
+            major_minor = '{0}.{1}'.format(match.groupdict()['major'],
+                                           match.groupdict()['minor'])
+            tarball = 'hdf5-{}.tar.bz2'.format(self.__version)
+            self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format(
+                self.__baseurl, major_minor, self.__version, tarball)
 
     def runtime(self, _from='0'):
         """Generate the set of instructions to install the runtime specific

hpccm/building_blocks/netcdf.py

@@ -23,6 +23,8 @@
 
 from packaging.version import Version
 import posixpath
+import re
+from copy import copy as _copy
 
 import hpccm.config
 import hpccm.templates.envvars
@@ -33,6 +35,7 @@
 from hpccm.building_blocks.packages import packages
 from hpccm.common import linux_distro
 from hpccm.primitives.comment import comment
+from hpccm.toolchain import toolchain
 
 class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
     """The `netcdf` building block downloads, configures, builds, and
@@ -80,10 +83,10 @@ class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
 
     ospackages: List of OS packages to install prior to configuring
     and building.  For Ubuntu, the default values are
-    `ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`,
-    `wget`, and `zlib1g-dev`.  For RHEL-based Linux distributions the
-    default values are `ca-certificates`, `file`, `libcurl-devel`
-    `m4`, `make`, `wget`, and `zlib-devel`.
+    `ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`,
+    `make`, `wget`, and `zlib1g-dev`.  For RHEL-based Linux distributions the
+    default values are `ca-certificates`, `file`, `libcurl-devel`,
+    `libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`.
 
     prefix: The top level install location.  The default location is
     `/usr/local/netcdf`.
@@ -134,6 +137,9 @@ def __init__(self, **kwargs):
         self.__ospackages = kwargs.pop('ospackages', [])
         self.__prefix = kwargs.pop('prefix', '/usr/local/netcdf')
         self.__runtime_ospackages = [] # Filled in by __distro()
+        # Create a copy of the toolchain so that it can be modified
+        # without impacting the original
+        self.__toolchain = _copy(kwargs.pop('toolchain', toolchain()))
         self.__version = kwargs.pop('version', '4.7.4')
         self.__version_cxx = kwargs.pop('version_cxx', '4.3.1')
         self.__version_fortran = kwargs.pop('version_fortran', '4.5.3')
@@ -165,6 +171,7 @@ def __init__(self, **kwargs):
             directory=self.__directory_c,
             prefix=self.__prefix,
             runtime_environment=self.environment_variables,
+            toolchain=self.__toolchain,
             url=self.__url_c,
             **kwargs)]
 
@@ -180,12 +187,19 @@ def __init__(self, **kwargs):
                 # Checks fail when using parallel make.  Disable it.
                 parallel=1 if self.__check else '$(nproc)',
                 prefix=self.__prefix,
+                toolchain=self.__toolchain,
                 url='{0}/v{1}.tar.gz'.format(self.__baseurl_cxx,
                                              self.__version_cxx),
                 **kwargs))
 
         # Setup optional Fortran build configuration
         if self.__fortran:
+            # PIC workaround when using the NVIDIA compilers
+            if self.__toolchain.FC and re.match('.*nvfortran',
+                                                self.__toolchain.FC):
+                if not self.__toolchain.FCFLAGS:
+                    self.__toolchain.FCFLAGS = '-fPIC'
+
             comments.append('NetCDF Fortran version {}'.format(self.__version_fortran))
             self.__bb.append(generic_autotools(
                 annotations={'version': self.__version_fortran},
@@ -196,6 +210,7 @@ def __init__(self, **kwargs):
                 # Checks fail when using parallel make.  Disable it.
                 parallel=1 if self.__check else '$(nproc)',
                 prefix=self.__prefix,
+                toolchain=self.__toolchain,
                 url='{0}/v{1}.tar.gz'.format(self.__baseurl_fortran,
                                              self.__version_fortran),
                 **kwargs))
@@ -212,14 +227,14 @@ def __distro(self):
         if hpccm.config.g_linux_distro == linux_distro.UBUNTU:
             if not self.__ospackages:
                 self.__ospackages = ['ca-certificates', 'file',
-                                     'libcurl4-openssl-dev', 'm4', 'make',
-                                     'wget', 'zlib1g-dev']
+                                     'libcurl4-openssl-dev', 'libxml2-dev',
+                                     'm4', 'make', 'wget', 'zlib1g-dev']
             self.__runtime_ospackages = ['zlib1g']
         elif hpccm.config.g_linux_distro == linux_distro.CENTOS:
             if not self.__ospackages:
                 self.__ospackages = ['ca-certificates', 'file',
-                                     'libcurl-devel', 'm4', 'make',
-                                     'wget', 'zlib-devel']
+                                     'libcurl-devel', 'libxml2-devel', 'm4',
+                                     'make', 'wget', 'zlib-devel']
                 if self.__check:
                     self.__ospackages.append('diffutils')
             self.__runtime_ospackages = ['zlib']

test/test_hdf5.py

@@ -36,8 +36,8 @@ def setUp(self):
     def test_defaults_ubuntu(self):
         """Default hdf5 building block"""
         h = hdf5()
-        self.assertEqual(str(h),
-r'''# HDF5 version 1.12.0
+        self.assertMultiLineEqual(str(h),
+r'''# HDF5 version 1.14.5
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         bzip2 \
@@ -46,12 +46,12 @@ def test_defaults_ubuntu(self):
         wget \
         zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \
-    cd /var/tmp/hdf5-1.12.0 &&   ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/hdf5-1.14.5 &&   ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2
+    rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz
 ENV CPATH=/usr/local/hdf5/include:$CPATH \
     HDF5_DIR=/usr/local/hdf5 \
     LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \
@@ -63,21 +63,21 @@ def test_defaults_ubuntu(self):
     def test_defaults_centos(self):
         """Default hdf5 building block"""
         h = hdf5()
-        self.assertEqual(str(h),
-r'''# HDF5 version 1.12.0
+        self.assertMultiLineEqual(str(h),
+r'''# HDF5 version 1.14.5
 RUN yum install -y \
         bzip2 \
         file \
         make \
         wget \
         zlib-devel && \
     rm -rf /var/cache/yum/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \
-    cd /var/tmp/hdf5-1.12.0 &&   ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/hdf5-1.14.5 &&   ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2
+    rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz
 ENV CPATH=/usr/local/hdf5/include:$CPATH \
     HDF5_DIR=/usr/local/hdf5 \
     LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \
@@ -89,7 +89,7 @@ def test_defaults_centos(self):
     def test_ldconfig(self):
         """ldconfig option"""
         h = hdf5(ldconfig=True, version='1.10.4')
-        self.assertEqual(str(h),
+        self.assertMultiLineEqual(str(h),
 r'''# HDF5 version 1.10.4
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -117,7 +117,7 @@ def test_runtime(self):
         """Runtime"""
         h = hdf5()
         r = h.runtime()
-        self.assertEqual(r,
+        self.assertMultiLineEqual(r,
 r'''# HDF5
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

test/test_netcdf.py

@@ -36,14 +36,15 @@ def setUp(self):
     def test_defaults_ubuntu(self):
         """Default netcdf building block"""
         n = netcdf()
-        self.assertEqual(str(n),
+        self.assertMultiLineEqual(str(n),
 r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran
 # version 4.5.3
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates \
         file \
         libcurl4-openssl-dev \
+        libxml2-dev \
         m4 \
         make \
         wget \
@@ -77,13 +78,14 @@ def test_defaults_ubuntu(self):
     def test_defaults_centos(self):
         """Default netcdf building block"""
         n = netcdf()
-        self.assertEqual(str(n),
+        self.assertMultiLineEqual(str(n),
 r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran
 # version 4.5.3
 RUN yum install -y \
         ca-certificates \
         file \
         libcurl-devel \
+        libxml2-devel \
         m4 \
         make \
         wget \
@@ -118,14 +120,15 @@ def test_ldconfig(self):
         """ldconfig option"""
         n = netcdf(ldconfig=True, version='4.6.1', version_cxx='4.3.0',
                    version_fortran='4.4.4')
-        self.assertEqual(str(n),
+        self.assertMultiLineEqual(str(n),
 r'''# NetCDF version 4.6.1, NetCDF C++ version 4.3.0, NetCDF Fortran
 # version 4.4.4
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates \
         file \
         libcurl4-openssl-dev \
+        libxml2-dev \
         m4 \
         make \
         wget \
@@ -162,7 +165,7 @@ def test_runtime(self):
         """Runtime"""
         n = netcdf()
         r = n.runtime()
-        self.assertEqual(r,
+        self.assertMultiLineEqual(r,
 r'''# NetCDF
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \