Merge pull request #2637 from docker/4.3.0-release

4.3.0 release

Author: aiordache

Jenkinsfile

@@ -31,7 +31,7 @@ def buildImages = { ->
 }
 
 def getDockerVersions = { ->
-  def dockerVersions = ["19.03.5"]
+  def dockerVersions = ["19.03.12"]
   wrappedNode(label: "amd64 && ubuntu-1804 && overlay2") {
     def result = sh(script: """docker run --rm \\
         --entrypoint=python \\
@@ -66,7 +66,7 @@ def runTests = { Map settings ->
     throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`")
   }
   if (!dockerVersion) {
-    throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '1.12.3')`")
+    throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`")
   }
   if (!pythonVersion) {
     throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py2.7')`")

Makefile

@@ -41,8 +41,8 @@ integration-test: build
 integration-test-py3: build-py3
 	docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file}
 
-TEST_API_VERSION ?= 1.35
-TEST_ENGINE_VERSION ?= 19.03.5
+TEST_API_VERSION ?= 1.39
+TEST_ENGINE_VERSION ?= 19.03.12
 
 .PHONY: setup-network
 setup-network:

docker/api/container.py

@@ -480,6 +480,9 @@ def create_host_config(self, *args, **kwargs):
                 For example, ``/dev/sda:/dev/xvda:rwm`` allows the container
                 to have read-write access to the host's ``/dev/sda`` via a
                 node named ``/dev/xvda`` inside the container.
+            device_requests (:py:class:`list`): Expose host resources such as
+                GPUs to the container, as a list of
+                :py:class:`docker.types.DeviceRequest` instances.
             dns (:py:class:`list`): Set custom DNS servers.
             dns_opt (:py:class:`list`): Additional options to be added to the
                 container's ``resolv.conf`` file
@@ -636,6 +639,8 @@ def create_endpoint_config(self, *args, **kwargs):
                 network, using the IPv6 protocol. Defaults to ``None``.
             link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6)
                 addresses.
+            driver_opt (dict): A dictionary of options to provide to the
+                network driver. Defaults to ``None``.
 
         Returns:
             (dict) An endpoint config.
@@ -694,7 +699,8 @@ def export(self, container, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
         return self._stream_raw_result(res, chunk_size, False)
 
     @utils.check_resource('container')
-    def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
+    def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE,
+                    encode_stream=False):
         """
         Retrieve a file or folder from a container in the form of a tar
         archive.
@@ -705,6 +711,8 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
             chunk_size (int): The number of bytes returned by each iteration
                 of the generator. If ``None``, data will be streamed as it is
                 received. Default: 2 MB
+            encode_stream (bool): Determines if data should be encoded
+                (gzip-compressed) during transmission. Default: False
 
         Returns:
             (tuple): First element is a raw tar data stream. Second element is
@@ -729,8 +737,13 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
         params = {
             'path': path
         }
+        headers = {
+            "Accept-Encoding": "gzip, deflate"
+        } if encode_stream else {
+            "Accept-Encoding": "identity"
+        }
         url = self._url('/containers/{0}/archive', container)
-        res = self._get(url, params=params, stream=True)
+        res = self._get(url, params=params, stream=True, headers=headers)
         self._raise_for_status(res)
         encoded_stat = res.headers.get('x-docker-container-path-stat')
         return (
@@ -1120,7 +1133,7 @@ def stats(self, container, decode=None, stream=True):
         else:
             if decode:
                 raise errors.InvalidArgument(
-                    "decode is only available in conjuction with stream=True"
+                    "decode is only available in conjunction with stream=True"
                 )
             return self._result(self._get(url, params={'stream': False}),
                                 json=True)

docker/api/network.py

@@ -216,7 +216,7 @@ def inspect_network(self, net_id, verbose=None, scope=None):
     def connect_container_to_network(self, container, net_id,
                                      ipv4_address=None, ipv6_address=None,
                                      aliases=None, links=None,
-                                     link_local_ips=None):
+                                     link_local_ips=None, driver_opt=None):
         """
         Connect a container to a network.
 
@@ -240,7 +240,8 @@ def connect_container_to_network(self, container, net_id,
             "Container": container,
             "EndpointConfig": self.create_endpoint_config(
                 aliases=aliases, links=links, ipv4_address=ipv4_address,
-                ipv6_address=ipv6_address, link_local_ips=link_local_ips
+                ipv6_address=ipv6_address, link_local_ips=link_local_ips,
+                driver_opt=driver_opt
             ),
         }
 

docker/constants.py

@@ -1,7 +1,7 @@
 import sys
 from .version import version
 
-DEFAULT_DOCKER_API_VERSION = '1.35'
+DEFAULT_DOCKER_API_VERSION = '1.39'
 MINIMUM_DOCKER_API_VERSION = '1.21'
 DEFAULT_TIMEOUT_SECONDS = 60
 STREAM_HEADER_SIZE_BYTES = 8

docker/models/containers.py

@@ -225,7 +225,8 @@ def export(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
         """
         return self.client.api.export(self.id, chunk_size)
 
-    def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
+    def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE,
+                    encode_stream=False):
         """
         Retrieve a file or folder from the container in the form of a tar
         archive.
@@ -235,6 +236,8 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
             chunk_size (int): The number of bytes returned by each iteration
                 of the generator. If ``None``, data will be streamed as it is
                 received. Default: 2 MB
+            encode_stream (bool): Determines if data should be encoded
+                (gzip-compressed) during transmission. Default: False
 
         Returns:
             (tuple): First element is a raw tar data stream. Second element is
@@ -255,7 +258,8 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
             ...    f.write(chunk)
             >>> f.close()
         """
-        return self.client.api.get_archive(self.id, path, chunk_size)
+        return self.client.api.get_archive(self.id, path,
+                                           chunk_size, encode_stream)
 
     def kill(self, signal=None):
         """
@@ -579,6 +583,9 @@ def run(self, image, command=None, stdout=True, stderr=False,
                 For example, ``/dev/sda:/dev/xvda:rwm`` allows the container
                 to have read-write access to the host's ``/dev/sda`` via a
                 node named ``/dev/xvda`` inside the container.
+            device_requests (:py:class:`list`): Expose host resources such as
+                GPUs to the container, as a list of
+                :py:class:`docker.types.DeviceRequest` instances.
             dns (:py:class:`list`): Set custom DNS servers.
             dns_opt (:py:class:`list`): Additional options to be added to the
                 container's ``resolv.conf`` file.
@@ -998,6 +1005,7 @@ def prune(self, filters=None):
     'device_write_bps',
     'device_write_iops',
     'devices',
+    'device_requests',
     'dns_opt',
     'dns_search',
     'dns',

docker/models/networks.py

@@ -46,6 +46,8 @@ def connect(self, container, *args, **kwargs):
                 network, using the IPv6 protocol. Defaults to ``None``.
             link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6)
                 addresses.
+            driver_opt (dict): A dictionary of options to provide to the
+                network driver. Defaults to ``None``.
 
         Raises:
             :py:class:`docker.errors.APIError`

docker/tls.py

@@ -32,7 +32,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None,
         # https://docs.docker.com/engine/articles/https/
         # This diverges from the Docker CLI in that users can specify 'tls'
         # here, but also disable any public/default CA pool verification by
-        # leaving tls_verify=False
+        # leaving verify=False
 
         self.assert_hostname = assert_hostname
         self.assert_fingerprint = assert_fingerprint
@@ -62,7 +62,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 # https://github.com/docker/docker-py/issues/963
                 self.ssl_version = ssl.PROTOCOL_TLSv1
 
-        # "tls" and "tls_verify" must have both or neither cert/key files In
+        # "client_cert" must have both or neither cert/key files. In
         # either case, Alert the user when both are expected, but any are
         # missing.
 
@@ -71,15 +71,15 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 tls_cert, tls_key = client_cert
             except ValueError:
                 raise errors.TLSParameterError(
-                    'client_config must be a tuple of'
+                    'client_cert must be a tuple of'
                     ' (client certificate, key file)'
                 )
 
             if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
                                               not os.path.isfile(tls_key)):
                 raise errors.TLSParameterError(
                     'Path to a certificate and key files must be provided'
-                    ' through the client_config param'
+                    ' through the client_cert param'
                 )
             self.cert = (tls_cert, tls_key)
 
@@ -88,7 +88,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None,
         self.ca_cert = ca_cert
         if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
             raise errors.TLSParameterError(
-                'Invalid CA certificate provided for `tls_ca_cert`.'
+                'Invalid CA certificate provided for `ca_cert`.'
             )
 
     def configure_client(self, client):

docker/types/__init__.py

@@ -1,5 +1,7 @@
 # flake8: noqa
-from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit
+from .containers import (
+    ContainerConfig, HostConfig, LogConfig, Ulimit, DeviceRequest
+)
 from .daemon import CancellableStream
 from .healthcheck import Healthcheck
 from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig

docker/types/containers.py

@@ -154,6 +154,104 @@ def hard(self, value):
         self['Hard'] = value
 
 
+class DeviceRequest(DictType):
+    """
+    Create a device request to be used with
+    :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`.
+
+    Args:
+
+        driver (str): Which driver to use for this device. Optional.
+        count (int): Number or devices to request. Optional.
+            Set to -1 to request all available devices.
+        device_ids (list): List of strings for device IDs. Optional.
+            Set either ``count`` or ``device_ids``.
+        capabilities (list): List of lists of strings to request
+            capabilities. Optional. The global list acts like an OR,
+            and the sub-lists are AND. The driver will try to satisfy
+            one of the sub-lists.
+            Available capabilities for the ``nvidia`` driver can be found
+            `here <https://github.com/NVIDIA/nvidia-container-runtime>`_.
+        options (dict): Driver-specific options. Optional.
+    """
+
+    def __init__(self, **kwargs):
+        driver = kwargs.get('driver', kwargs.get('Driver'))
+        count = kwargs.get('count', kwargs.get('Count'))
+        device_ids = kwargs.get('device_ids', kwargs.get('DeviceIDs'))
+        capabilities = kwargs.get('capabilities', kwargs.get('Capabilities'))
+        options = kwargs.get('options', kwargs.get('Options'))
+
+        if driver is None:
+            driver = ''
+        elif not isinstance(driver, six.string_types):
+            raise ValueError('DeviceRequest.driver must be a string')
+        if count is None:
+            count = 0
+        elif not isinstance(count, int):
+            raise ValueError('DeviceRequest.count must be an integer')
+        if device_ids is None:
+            device_ids = []
+        elif not isinstance(device_ids, list):
+            raise ValueError('DeviceRequest.device_ids must be a list')
+        if capabilities is None:
+            capabilities = []
+        elif not isinstance(capabilities, list):
+            raise ValueError('DeviceRequest.capabilities must be a list')
+        if options is None:
+            options = {}
+        elif not isinstance(options, dict):
+            raise ValueError('DeviceRequest.options must be a dict')
+
+        super(DeviceRequest, self).__init__({
+            'Driver': driver,
+            'Count': count,
+            'DeviceIDs': device_ids,
+            'Capabilities': capabilities,
+            'Options': options
+        })
+
+    @property
+    def driver(self):
+        return self['Driver']
+
+    @driver.setter
+    def driver(self, value):
+        self['Driver'] = value
+
+    @property
+    def count(self):
+        return self['Count']
+
+    @count.setter
+    def count(self, value):
+        self['Count'] = value
+
+    @property
+    def device_ids(self):
+        return self['DeviceIDs']
+
+    @device_ids.setter
+    def device_ids(self, value):
+        self['DeviceIDs'] = value
+
+    @property
+    def capabilities(self):
+        return self['Capabilities']
+
+    @capabilities.setter
+    def capabilities(self, value):
+        self['Capabilities'] = value
+
+    @property
+    def options(self):
+        return self['Options']
+
+    @options.setter
+    def options(self, value):
+        self['Options'] = value
+
+
 class HostConfig(dict):
     def __init__(self, version, binds=None, port_bindings=None,
                  lxc_conf=None, publish_all_ports=False, links=None,
@@ -176,7 +274,7 @@ def __init__(self, version, binds=None, port_bindings=None,
                  volume_driver=None, cpu_count=None, cpu_percent=None,
                  nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None,
                  cpu_rt_period=None, cpu_rt_runtime=None,
-                 device_cgroup_rules=None):
+                 device_cgroup_rules=None, device_requests=None):
 
         if mem_limit is not None:
             self['Memory'] = parse_bytes(mem_limit)
@@ -536,6 +634,19 @@ def __init__(self, version, binds=None, port_bindings=None,
                 )
             self['DeviceCgroupRules'] = device_cgroup_rules
 
+        if device_requests is not None:
+            if version_lt(version, '1.40'):
+                raise host_config_version_error('device_requests', '1.40')
+            if not isinstance(device_requests, list):
+                raise host_config_type_error(
+                    'device_requests', device_requests, 'list'
+                )
+            self['DeviceRequests'] = []
+            for req in device_requests:
+                if not isinstance(req, DeviceRequest):
+                    req = DeviceRequest(**req)
+                self['DeviceRequests'].append(req)
+
 
 def host_config_type_error(param, param_value, expected):
     error_msg = 'Invalid type for {0} param: expected {1} but found {2}'

docker/types/networks.py

@@ -4,7 +4,7 @@
 
 class EndpointConfig(dict):
     def __init__(self, version, aliases=None, links=None, ipv4_address=None,
-                 ipv6_address=None, link_local_ips=None):
+                 ipv6_address=None, link_local_ips=None, driver_opt=None):
         if version_lt(version, '1.22'):
             raise errors.InvalidVersion(
                 'Endpoint config is not supported for API version < 1.22'
@@ -33,6 +33,15 @@ def __init__(self, version, aliases=None, links=None, ipv4_address=None,
         if ipam_config:
             self['IPAMConfig'] = ipam_config
 
+        if driver_opt:
+            if version_lt(version, '1.32'):
+                raise errors.InvalidVersion(
+                    'DriverOpts is not supported for API version < 1.32'
+                )
+            if not isinstance(driver_opt, dict):
+                raise TypeError('driver_opt must be a dictionary')
+            self['DriverOpts'] = driver_opt
+
 
 class NetworkingConfig(dict):
     def __init__(self, endpoints_config=None):

docker/version.py

@@ -1,2 +1,2 @@
-version = "4.2.2"
+version = "4.3.0"
 version_info = tuple([int(d) for d in version.split("-")[0].split(".")])