From 19a501fdfc8a86cac549d64cf43ddb1fedb76a71 Mon Sep 17 00:00:00 2001 From: "alan.souza" Date: Thu, 17 Dec 2020 12:41:05 -0300 Subject: [PATCH 1/4] add the docker specific arg primitive. This primitive return an empty string for the shell and singularity container types --- hpccm/primitives/__init__.py | 3 +- hpccm/primitives/arg.py | 70 +++++++++++++++++++++ test/test_arg.py | 119 +++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 hpccm/primitives/arg.py create mode 100644 test/test_arg.py diff --git a/hpccm/primitives/__init__.py b/hpccm/primitives/__init__.py index 92addf38..cc7bbcf1 100644 --- a/hpccm/primitives/__init__.py +++ b/hpccm/primitives/__init__.py @@ -15,7 +15,7 @@ from __future__ import absolute_import __all__ = ['baseimage', 'blob', 'comment', 'copy', 'environment', 'label', - 'raw', 'runscript', 'shell', 'user', 'workdir'] + 'raw', 'runscript', 'shell', 'user', 'workdir', 'arg'] from hpccm.primitives.baseimage import baseimage from hpccm.primitives.blob import blob @@ -28,3 +28,4 @@ from hpccm.primitives.shell import shell from hpccm.primitives.user import user from hpccm.primitives.workdir import workdir +from hpccm.primitives.arg import arg diff --git a/hpccm/primitives/arg.py b/hpccm/primitives/arg.py new file mode 100644 index 00000000..92775b7a --- /dev/null +++ b/hpccm/primitives/arg.py @@ -0,0 +1,70 @@ +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name, too-few-public-methods + +"""Environment primitive""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import logging # pylint: disable=unused-import + +import hpccm.config + +from hpccm.common import container_type + +class arg(object): + """The `arg` primitive sets the corresponding environment + variables during the build time of a docker container. + This primitive is docker specific and is ignored for singularity + + # Parameters + + variables: A dictionary of key / value pairs. The default is an + empty dictionary. + + # Examples + + ```python + arg(variables={'HTTP_PROXY': 'proxy.example.com', 'NO_PROXY':'example.com'}) + ``` + """ + + def __init__(self, **kwargs): + """Initialize primitive""" + self.__variables = kwargs.get('variables', {}) + + def __str__(self): + """String representation of the primitive""" + if self.__variables: + + if hpccm.config.g_ctype == container_type.SINGULARITY or \ + hpccm.config.g_ctype == container_type.BASH: + return "" + elif hpccm.config.g_ctype == container_type.DOCKER: + string = "" + num_vars = len(self.__variables) + for count, (key, val) in enumerate(sorted(self.__variables.items())): + eol = "" if count == num_vars - 1 else "\n" + if val == "": + string += 'ARG {0}'.format(key) + eol + else: + string += 'ARG {0}={1}'.format(key, val) + eol + return string + else: + raise RuntimeError('Unknown container type') + else: + return '' diff --git a/test/test_arg.py b/test/test_arg.py new file mode 100644 index 00000000..fae542c8 --- /dev/null +++ b/test/test_arg.py @@ -0,0 +1,119 @@ +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name, too-few-public-methods, bad-continuation + +"""Test cases for the arg module""" + +from __future__ import unicode_literals +from __future__ import print_function + +import logging # pylint: disable=unused-import +import unittest + +from helpers import bash, docker, invalid_ctype, singularity + +from hpccm.primitives.arg import arg + +class Test_arg(unittest.TestCase): + def setUp(self): + """Disable logging output messages""" + logging.disable(logging.ERROR) + + @docker + def test_empty(self): + """No arg specified""" + e = arg() + self.assertEqual(str(e), '') + + @invalid_ctype + def test_invalid_ctype(self): + """Invalid container type specified""" + e = arg(variables={'A': 'B'}) + with self.assertRaises(RuntimeError): + str(e) + + @docker + def test_single_docker(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e), 'ARG A=B') + + @docker + def test_single_docker_nodefault(self): + """Single arg variable specified (no default value)""" + e = arg(variables={'A': ''}) + self.assertEqual(str(e), 'ARG A') + + @singularity + def test_single_singularity(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e), '') + + @bash + def test_single_bash(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e), '') + + @docker + def test_single_export_docker(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e), 'ARG A=B') + + @singularity + def test_single_export_singularity(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e),'') + + @bash + def test_single_export_bash(self): + """Single arg variable specified""" + e = arg(variables={'A': 'B'}) + self.assertEqual(str(e), '') + + @docker + def test_multiple_docker(self): + """Multiple arg variables specified""" + e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}) + self.assertEqual(str(e), +'''ARG ONE=1 +ARG THREE=3 +ARG TWO=2''') + + @docker + def test_multiple_docker_nodefault(self): + """Multiple arg variables specified (no default value)""" + e = arg(variables={'ONE': '', 'TWO': '', 'THREE': ''}) + self.assertEqual(str(e), +'''ARG ONE +ARG THREE +ARG TWO''') + + @singularity + def test_multiple_singularity(self): + """Multiple arg variables specified""" + e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}, + _export=False) + self.assertEqual(str(e),'') + + @bash + def test_multiple_bash(self): + """Multiple arg variables specified""" + e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}, + _export=False) + self.assertEqual(str(e),'') \ No newline at end of file From e84a25907f231f455cacfe3a501335bec92f36c1 Mon Sep 17 00:00:00 2001 From: "alan.souza" Date: Thu, 17 Dec 2020 12:53:44 -0300 Subject: [PATCH 2/4] documentation for the arg primitive --- docs/primitives.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/primitives.md b/docs/primitives.md index c61a253c..352a7935 100644 --- a/docs/primitives.md +++ b/docs/primitives.md @@ -199,6 +199,27 @@ __Examples__ environment(variables={'PATH': '/usr/local/bin:$PATH'}) ``` +# arg +```python +arg(self, **kwargs) +``` + +The `arg` primitive sets the corresponding environment variables. +These variables could have a default value or not. For the former +case the value should be especified on the command line used to +build the image. This primitive is specific to the Docker container. +For the Singularity and bash container this primitive return only a +empry string. + +- __variables__: A dictionary of key / value pairs. The default is an +empty dictionary. + +__Examples__ + +```python +arg(variables={'HTTP_PROXY':'proxy.example.com', 'NO_PROXY':'example.com'}) +``` + # label ```python label(self, **kwargs) From 8f6b4830db15a9c5c6edd23dd303c58bc0e1a615 Mon Sep 17 00:00:00 2001 From: "alan.souza" Date: Fri, 18 Dec 2020 05:58:57 -0300 Subject: [PATCH 3/4] change the arg primitive to work with singularity and bash containers --- docs/primitives.md | 23 +++++++++---- hpccm/primitives/__init__.py | 4 +-- hpccm/primitives/arg.py | 45 ++++++++++++++++++------- test/test_arg.py | 64 +++++++++++++++++++++++------------- 4 files changed, 94 insertions(+), 42 deletions(-) diff --git a/docs/primitives.md b/docs/primitives.md index 352a7935..5c1604e6 100644 --- a/docs/primitives.md +++ b/docs/primitives.md @@ -204,12 +204,12 @@ environment(variables={'PATH': '/usr/local/bin:$PATH'}) arg(self, **kwargs) ``` -The `arg` primitive sets the corresponding environment variables. -These variables could have a default value or not. For the former -case the value should be especified on the command line used to -build the image. This primitive is specific to the Docker container. -For the Singularity and bash container this primitive return only a -empry string. +The `arg` primitive sets the corresponding environment +variables during the build time of a docker container. +Singularity and "bash" containers does not have a strict version of the +ARG keyword found on Dockerfiles but is possible to simulate +the behavior of this keyword as a build time parameter for the +Singularity and bash containers using environment variables. - __variables__: A dictionary of key / value pairs. The default is an empty dictionary. @@ -220,6 +220,17 @@ __Examples__ arg(variables={'HTTP_PROXY':'proxy.example.com', 'NO_PROXY':'example.com'}) ``` +```bash + SINGULARITYENV_HTTP_PROXY="proxy.example.com" \ + SINGULARITYENV_NO_PROXY="example.com \ + singularity build image.sif recipe.def" +``` + +```bash + HTTP_PROXY="proxy.example.com" \ + NO_PROXY="example.com \ +``` + # label ```python label(self, **kwargs) diff --git a/hpccm/primitives/__init__.py b/hpccm/primitives/__init__.py index cc7bbcf1..0c5c3d55 100644 --- a/hpccm/primitives/__init__.py +++ b/hpccm/primitives/__init__.py @@ -14,8 +14,8 @@ from __future__ import absolute_import -__all__ = ['baseimage', 'blob', 'comment', 'copy', 'environment', 'label', - 'raw', 'runscript', 'shell', 'user', 'workdir', 'arg'] +__all__ = ['arg', 'baseimage', 'blob', 'comment', 'copy', 'environment', 'label', + 'raw', 'runscript', 'shell', 'user', 'workdir'] from hpccm.primitives.baseimage import baseimage from hpccm.primitives.blob import blob diff --git a/hpccm/primitives/arg.py b/hpccm/primitives/arg.py index 92775b7a..5237c344 100644 --- a/hpccm/primitives/arg.py +++ b/hpccm/primitives/arg.py @@ -14,7 +14,7 @@ # pylint: disable=invalid-name, too-few-public-methods -"""Environment primitive""" +"""Arg primitive""" from __future__ import absolute_import from __future__ import unicode_literals @@ -29,7 +29,10 @@ class arg(object): """The `arg` primitive sets the corresponding environment variables during the build time of a docker container. - This primitive is docker specific and is ignored for singularity + Singularity and "bash" containers does not have a strict version of the + ARG keyword found on Dockerfiles but is possible to simulate + the behavior of this keyword as a build time parameter for the + Singularity and bash containers using environment variables. # Parameters @@ -39,10 +42,21 @@ class arg(object): # Examples ```python - arg(variables={'HTTP_PROXY': 'proxy.example.com', 'NO_PROXY':'example.com'}) + arg(variables={'HTTP_PROXY':'proxy.example.com', 'NO_PROXY':'example.com'}) + + ```bash + SINGULARITYENV_HTTP_PROXY="proxy.example.com" \ + SINGULARITYENV_NO_PROXY="example.com \ + singularity build image.sif recipe.def" ``` - """ + ```bash + HTTP_PROXY="proxy.example.com" \ + NO_PROXY="example.com \ + recipe.sh" + ``` + + """ def __init__(self, **kwargs): """Initialize primitive""" self.__variables = kwargs.get('variables', {}) @@ -50,14 +64,23 @@ def __init__(self, **kwargs): def __str__(self): """String representation of the primitive""" if self.__variables: - - if hpccm.config.g_ctype == container_type.SINGULARITY or \ - hpccm.config.g_ctype == container_type.BASH: - return "" + string = "" + num_vars = len(self.__variables) + variables = self.__variables + if hpccm.config.g_ctype == container_type.SINGULARITY: + if num_vars > 0: + string += "%post" + "\n" + for count, (key, val) in enumerate(sorted(variables.items())): + eol = "" if count == num_vars - 1 else "\n" + string += ' {0}=${{{0}:-"{1}"}}'.format(key, val) + eol + return string + elif hpccm.config.g_ctype == container_type.BASH: + for count, (key, val) in enumerate(sorted(variables.items())): + eol = "" if count == num_vars - 1 else "\n" + string += '{0}=${{{0}:-"{1}"}}'.format(key, val) + eol + return string elif hpccm.config.g_ctype == container_type.DOCKER: - string = "" - num_vars = len(self.__variables) - for count, (key, val) in enumerate(sorted(self.__variables.items())): + for count, (key, val) in enumerate(sorted(variables.items())): eol = "" if count == num_vars - 1 else "\n" if val == "": string += 'ARG {0}'.format(key) + eol diff --git a/test/test_arg.py b/test/test_arg.py index fae542c8..c9306362 100644 --- a/test/test_arg.py +++ b/test/test_arg.py @@ -60,31 +60,25 @@ def test_single_docker_nodefault(self): def test_single_singularity(self): """Single arg variable specified""" e = arg(variables={'A': 'B'}) - self.assertEqual(str(e), '') - - @bash - def test_single_bash(self): - """Single arg variable specified""" - e = arg(variables={'A': 'B'}) - self.assertEqual(str(e), '') + self.assertEqual(str(e), '%post\n A=${A:-"B"}') - @docker - def test_single_export_docker(self): + @singularity + def test_single_singularity_nodefault(self): """Single arg variable specified""" - e = arg(variables={'A': 'B'}) - self.assertEqual(str(e), 'ARG A=B') + e = arg(variables={'A': ''}) + self.assertEqual(str(e), '%post\n A=${A:-""}') - @singularity - def test_single_export_singularity(self): + @bash + def test_single_bash(self): """Single arg variable specified""" e = arg(variables={'A': 'B'}) - self.assertEqual(str(e),'') + self.assertEqual(str(e), 'A=${A:-"B"}') @bash - def test_single_export_bash(self): + def test_single_bash_nodefault(self): """Single arg variable specified""" - e = arg(variables={'A': 'B'}) - self.assertEqual(str(e), '') + e = arg(variables={'A': ''}) + self.assertEqual(str(e), 'A=${A:-""}') @docker def test_multiple_docker(self): @@ -107,13 +101,37 @@ def test_multiple_docker_nodefault(self): @singularity def test_multiple_singularity(self): """Multiple arg variables specified""" - e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}, - _export=False) - self.assertEqual(str(e),'') + e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}) + self.assertEqual(str(e), +'''%post + ONE=${ONE:-"1"} + THREE=${THREE:-"3"} + TWO=${TWO:-"2"}''') + + @singularity + def test_multiple_singularity_nodefault(self): + """Multiple arg variables specified""" + e = arg(variables={'ONE':"", 'TWO':"", 'THREE':""}) + self.assertEqual(str(e), +'''%post + ONE=${ONE:-""} + THREE=${THREE:-""} + TWO=${TWO:-""}''') @bash def test_multiple_bash(self): """Multiple arg variables specified""" - e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}, - _export=False) - self.assertEqual(str(e),'') \ No newline at end of file + e = arg(variables={'ONE': 1, 'TWO': 2, 'THREE': 3}) + self.assertEqual(str(e), +'''ONE=${ONE:-"1"} +THREE=${THREE:-"3"} +TWO=${TWO:-"2"}''') + + @bash + def test_multiple_bash_nodefault(self): + """Multiple arg variables specified""" + e = arg(variables={'ONE': "", 'TWO': "", 'THREE': ""}) + self.assertEqual(str(e), +'''ONE=${ONE:-""} +THREE=${THREE:-""} +TWO=${TWO:-""}''') \ No newline at end of file From 910a56abcd833b3c68515847cac67274588d3f32 Mon Sep 17 00:00:00 2001 From: "alan.souza" Date: Fri, 18 Dec 2020 06:02:11 -0300 Subject: [PATCH 4/4] fix missing recipe.sh --- docs/primitives.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/primitives.md b/docs/primitives.md index 5c1604e6..747ca7c5 100644 --- a/docs/primitives.md +++ b/docs/primitives.md @@ -229,6 +229,7 @@ arg(variables={'HTTP_PROXY':'proxy.example.com', 'NO_PROXY':'example.com'}) ```bash HTTP_PROXY="proxy.example.com" \ NO_PROXY="example.com \ + recipe.sh ``` # label